{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "86a4bd36",
   "metadata": {},
   "source": [
    "\n",
    "\n",
    "下面展示基于隐马尔科夫模型的序列标注监督学习的代码。这里以命名实体识别任务为例，所使用的数据是Books数据集。为简单起见，标签序列采用BIO格式。首先构建数据集和标签集合："
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "41673bbb",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'B-NAME': 1, 'I-ORG': 2, 'B-PRO': 3, 'B-EDU': 4, 'I-NAME': 5, 'B-LOC': 6, 'B-TITLE': 7, 'B-RACE': 8, 'I-TITLE': 9, 'I-RACE': 10, 'I-PRO': 11, 'B-CONT': 12, 'I-EDU': 13, 'I-LOC': 14, 'I-CONT': 15, 'B-ORG': 16, 'O': 0}\n"
     ]
    }
   ],
   "source": [
    "import os\n",
    "import json\n",
    "from collections import defaultdict\n",
    "\n",
    "\n",
    "class NERDataset:\n",
    "    def __init__(self):\n",
    "        train_file, test_file, label_file = 'ner_train.jsonl',\\\n",
    "            'ner_test.jsonl', 'ner_labels.json'\n",
    "        \n",
    "        def read_file(file_name):\n",
    "            with open(file_name, 'r', encoding='utf-8') as fin:\n",
    "                json_list = list(fin)\n",
    "            data_split = []\n",
    "            for json_str in json_list:\n",
    "                raw = json.loads(json_str)\n",
    "                d = {'tokens': raw['tokens'], 'tags': raw['tags']}\n",
    "                data_split.append(d)\n",
    "            return data_split\n",
    "        \n",
    "        # 读取JSON文件，转化为Python对象\n",
    "        self.train_data, self.test_data = read_file(train_file),\\\n",
    "            read_file(test_file)\n",
    "        self.label2id = json.loads(open(label_file, 'r').read())\n",
    "        self.id2label = {}\n",
    "        for k, v in self.label2id.items():\n",
    "            self.id2label[v] = k\n",
    "        print(self.label2id)\n",
    "\n",
    "    # 建立词表，过滤低频词\n",
    "    def build_vocab(self, min_freq=3):\n",
    "        # 统计词频\n",
    "        frequency = defaultdict(int)\n",
    "        for data in self.train_data:\n",
    "            tokens = data['tokens']\n",
    "            for token in tokens:\n",
    "                frequency[token] += 1\n",
    "                \n",
    "        print(f'unique tokens = {len(frequency)}, '+\\\n",
    "              f'total counts = {sum(frequency.values())}, '+\\\n",
    "              f'max freq = {max(frequency.values())}, '+\\\n",
    "              f'min freq = {min(frequency.values())}')\n",
    "        \n",
    "        # 由于词与标签一一对应，不能随便删除字符，\n",
    "        # 因此加入<unk>用于替代未登录词，加入<pad>用于批量训练\n",
    "        self.token2id = {'<pad>': 0, '<unk>': 1}\n",
    "        self.id2token = {0: '<pad>', 1: '<unk>'}\n",
    "        total_count = 0\n",
    "        # 将高频词加入词表\n",
    "        for token, freq in sorted(frequency.items(),\\\n",
    "                key=lambda x: -x[1]):\n",
    "            if freq > min_freq:\n",
    "                self.token2id[token] = len(self.token2id)\n",
    "                self.id2token[len(self.id2token)] = token\n",
    "                total_count += freq\n",
    "            else:\n",
    "                break\n",
    "        print(f'min_freq = {min_freq}, '\n",
    "              f'remaining tokens = {len(self.token2id)}, '\n",
    "              f'in-vocab rate = {total_count / sum(frequency.values())}')\n",
    "\n",
    "    # 将文本输入转化为词表中对应的索引\n",
    "    def convert_tokens_to_ids(self):\n",
    "        for data_split in [self.train_data, self.test_data]:\n",
    "            for data in data_split:\n",
    "                data['token_ids'] = []\n",
    "                for token in data['tokens']:\n",
    "                    data['token_ids'].append(self.token2id.get(token, 1)) \n",
    "        \n",
    "dataset = NERDataset()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "25650a7c",
   "metadata": {},
   "source": [
    "接下来建立词表，将词元转换为索引，并将数据转化为适合训练的格式。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "a3eb49e5",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "unique tokens = 2454, total counts = 182068, max freq = 5630, min freq = 1\n",
      "min_freq = 0, remaining tokens = 2456, in-vocab rate = 1.0\n",
      "[595, 510, 353, 263, 424, 225, 764, 637, 353, 65, 80, 47, 41, 74, 53, 41, 141, 23, 74, 7, 259, 27, 135, 47, 31, 74, 67, 25, 1605, 815, 1157, 225, 595, 1158, 738, 688, 353, 1025, 65, 234, 27, 135, 316, 41, 31, 7, 80, 41, 42, 31, 47, 27, 528, 41, 141, 27, 135, 67, 1005, 1606, 363, 19, 677, 294, 37, 678, 1756, 604, 873, 1006, 1005, 353, 33, 30, 9, 113, 5, 45, 11, 82, 60, 21, 26, 5, 186, 153, 100, 565, 873, 353, 424, 1993, 387, 346, 250, 13, 5, 102, 214, 25, 11, 82, 60, 21, 26, 5, 186, 2, 7, 459, 97, 89, 147, 140, 139, 69, 46, 21, 12, 2, 7, 964, 98, 240, 677, 59, 9, 103, 11, 82, 60, 335, 200, 76, 26, 6, 110, 20, 9, 15, 4, 122, 539, 241, 99, 621, 40, 200, 57, 82, 156, 25, 11, 82, 381, 371, 91, 202, 6, 52, 2, 7, 670, 100, 84, 204, 53, 120, 27, 42, 3, 56, 1159, 3, 329, 31, 265, 31, 3, 56, 394, 394, 3, 84, 357, 84, 7, 25, 397, 7, 41, 7, 74, 7, 135, 7, 31, 7, 87, 7, 259, 7, 31, 7, 74, 7, 41, 7, 131, 7, 28, 289, 385, 4]\n",
      "['阿', '里', '斯', '提', '德', '·', '波', '拉', '斯', '（', 'A', 'r', 'i', 's', 't', 'i', 'd', 'e', 's', ' ', 'B', 'o', 'u', 'r', 'a', 's', '）', '和', '卢', '卡', '雅', '·', '阿', '伊', '纳', '罗', '斯', '托', '（', 'L', 'o', 'u', 'k', 'i', 'a', ' ', 'A', 'i', 'n', 'a', 'r', 'o', 'z', 'i', 'd', 'o', 'u', '）', '夫', '妇', '二', '人', '均', '拥', '有', '希', '腊', '比', '雷', '埃', '夫', '斯', '技', '术', '教', '育', '学', '院', '计', '算', '机', '工', '程', '学', '位', '以', '及', '色', '雷', '斯', '德', '谟', '克', '利', '特', '大', '学', '电', '子', '和', '计', '算', '机', '工', '程', '学', '位', '，', ' ', '都', '从', '事', '过', '软', '件', '开', '发', '工', '作', '，', ' ', '且', '目', '前', '均', '为', '教', '授', '计', '算', '机', '相', '关', '课', '程', '的', '高', '中', '教', '师', '。', '他', '们', '写', '了', '很', '多', '关', '于', '算', '法', '和', '计', '算', '思', '维', '方', '面', '的', '书', '，', ' ', '涉', '及', 'P', 'y', 't', 'h', 'o', 'n', '、', 'C', '#', '、', 'J', 'a', 'v', 'a', '、', 'C', '+', '+', '、', 'P', 'H', 'P', ' ', '和', 'V', ' ', 'i', ' ', 's', ' ', 'u', ' ', 'a', ' ', 'l', ' ', 'B', ' ', 'a', ' ', 's', ' ', 'i', ' ', 'c', ' ', '等', '语', '言', '。']\n"
     ]
    }
   ],
   "source": [
    "# 截取一部分数据便于更快训练\n",
    "dataset.train_data = dataset.train_data[:1000]\n",
    "dataset.test_data = dataset.test_data[:200]\n",
    "\n",
    "dataset.build_vocab(min_freq=0)\n",
    "dataset.convert_tokens_to_ids()\n",
    "print(dataset.train_data[0]['token_ids'])\n",
    "print([dataset.id2token[token_id] for token_id in \\\n",
    "       dataset.train_data[0]['token_ids']])\n",
    "\n",
    "def collect_data(data_split):\n",
    "    X, Y = [], []\n",
    "    for data in data_split:\n",
    "        assert len(data['token_ids']) == len(data['tags'])\n",
    "        X.append(data['token_ids'])\n",
    "        Y.append(data['tags'])\n",
    "    return X, Y\n",
    "\n",
    "train_X, train_Y = collect_data(dataset.train_data)\n",
    "test_X, test_Y = collect_data(dataset.test_data)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "7de17192",
   "metadata": {},
   "source": [
    "随后实现隐马尔科夫模型，使用最大似然估计得到模型参数，使用维特比算法进行解码。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "3229fd6a",
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "\n",
    "class HMM:\n",
    "    def __init__(self, n_tags, n_tokens):\n",
    "        self.n_tags = n_tags\n",
    "        self.n_tokens = n_tokens\n",
    "    \n",
    "    # 使用最大似然估计计算模型参数\n",
    "    def fit(self, X, Y):\n",
    "        Y0_cnt = np.zeros(self.n_tags)\n",
    "        YY_cnt = np.zeros((self.n_tags, self.n_tags))\n",
    "        YX_cnt = np.zeros((self.n_tags, self.n_tokens))\n",
    "        for x, y in zip(X, Y):\n",
    "            Y0_cnt[y[0]] += 1\n",
    "            last_y = y[0]\n",
    "            for i in range(1, len(y)):\n",
    "                YY_cnt[last_y, y[i]] += 1\n",
    "                last_y = y[i]\n",
    "            for xi, yi in zip(x, y):\n",
    "                YX_cnt[yi, xi] += 1\n",
    "        self.init_prob = Y0_cnt / Y0_cnt.sum()\n",
    "        self.transition_prob = YY_cnt\n",
    "        self.emission_prob = YX_cnt\n",
    "        for i in range(self.n_tags):\n",
    "            # 为了避免训练集过小时除0\n",
    "            yy_sum = YY_cnt[i].sum()\n",
    "            if yy_sum > 0:\n",
    "                self.transition_prob[i] = YY_cnt[i] / yy_sum\n",
    "            yx_sum = YX_cnt[i].sum()\n",
    "            if yx_sum > 0:\n",
    "                self.emission_prob[i] = YX_cnt[i] / yx_sum\n",
    "    \n",
    "    # 已知模型参数的条件下，使用维特比算法解码得到最优标签序列\n",
    "    def viterbi(self, x):\n",
    "        assert hasattr(self, 'init_prob') and hasattr(self,\\\n",
    "            'transition_prob') and hasattr(self, 'emission_prob')\n",
    "        Pi = np.zeros((len(x), self.n_tags))\n",
    "        Y = np.zeros((len(x), self.n_tags), dtype=np.int32)\n",
    "        # 初始化\n",
    "        for i in range(self.n_tags):\n",
    "            Pi[0, i] = self.init_prob[i] * self.emission_prob[i, x[0]]\n",
    "            Y[0, i] = -1\n",
    "        for t in range(1, len(x)):\n",
    "            for i in range(self.n_tags):\n",
    "                tmp = []\n",
    "                for j in range(self.n_tags):\n",
    "                    tmp.append(self.transition_prob[j, i] * Pi[t-1, j])\n",
    "                best_j = np.argmax(tmp)\n",
    "                # 维特比算法递推公式\n",
    "                Pi[t, i] = self.emission_prob[i, x[t]] * tmp[best_j]\n",
    "                Y[t, i] = best_j\n",
    "        y = [np.argmax(Pi[-1])]\n",
    "        for t in range(len(x)-1, 0, -1):\n",
    "            y.append(Y[t, y[-1]])\n",
    "        return np.max(Pi[len(x)-1]), y[::-1]\n",
    "    \n",
    "    def decode(self, X):\n",
    "        Y = []\n",
    "        for x in X:\n",
    "            _, y = self.viterbi(x)\n",
    "            Y.append(y)\n",
    "        return Y\n",
    "\n",
    "hmm = HMM(len(dataset.label2id), len(dataset.token2id))\n",
    "hmm.fit(train_X, train_Y)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "32868ed6",
   "metadata": {},
   "source": [
    "最后验证模型效果"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "4f4b7e88",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "precision = 0.45762149610217284, recall = 0.3289222699093944, f1 = 0.38274259554692375\n",
      "precision = 0.4189636163175303, recall = 0.23270055113288426, f1 = 0.2992125984251969\n"
     ]
    }
   ],
   "source": [
    "def extract_entity(labels):\n",
    "    entity_list = []\n",
    "    entity_start = -1\n",
    "    entity_length = 0\n",
    "    entity_type = None\n",
    "    for token_index, label in enumerate(labels):\n",
    "        if label.startswith('B'):\n",
    "            if entity_start != -1:\n",
    "                # 遇到了一个新的B，将上一个实体加入列表\n",
    "                entity_list.append((entity_start, entity_length,\\\n",
    "                    entity_type))\n",
    "            # 记录新实体\n",
    "            entity_start = token_index\n",
    "            entity_length = 1\n",
    "            entity_type = label.split('-')[1]\n",
    "        elif label.startswith('I'):\n",
    "            if entity_start != -1:\n",
    "                # 上一个实体未关闭，遇到了一个新的I\n",
    "                if entity_type == label.split('-')[1]:\n",
    "                    # 若上一个实体与当前类型相同，长度+1\n",
    "                    entity_length += 1\n",
    "                else:\n",
    "                    # 若上一个实体与当前类型不同，\n",
    "                    # 将上一个实体加入列表，重置实体\n",
    "                    entity_list.append((entity_start, entity_length,\\\n",
    "                        entity_type))\n",
    "                    entity_start = -1\n",
    "                    entity_length = 0\n",
    "                    entity_type = None\n",
    "        else:\n",
    "            if entity_start != -1:\n",
    "                # 遇到了一个新的O，将上一个实体加入列表\n",
    "                entity_list.append((entity_start, entity_length,\\\n",
    "                    entity_type))\n",
    "            # 重置实体\n",
    "            entity_start = -1\n",
    "            entity_length = 0\n",
    "            entity_type = None\n",
    "    if entity_start != -1:\n",
    "        # 将上一个实体加入列表\n",
    "        entity_list.append((entity_start, entity_length, entity_type))\n",
    "    return entity_list\n",
    "\n",
    "def compute_metric(Y, P):\n",
    "    true_entity_set = set()\n",
    "    pred_entity_set = set()\n",
    "    for sent_no, labels in enumerate(Y):\n",
    "        labels = [dataset.id2label[x] for x in labels]\n",
    "        for ent in extract_entity(labels):\n",
    "            true_entity_set.add((sent_no, ent))\n",
    "    for sent_no, labels in enumerate(P):\n",
    "        labels = [dataset.id2label[x] for x in labels]\n",
    "        for ent in extract_entity(labels):\n",
    "            pred_entity_set.add((sent_no, ent))\n",
    "    if len(true_entity_set) > 0:\n",
    "        recall = len(true_entity_set & pred_entity_set)\\\n",
    "            / len(true_entity_set)\n",
    "    else:\n",
    "        recall = 0\n",
    "    if len(pred_entity_set) > 0:\n",
    "        precision = len(true_entity_set & pred_entity_set)\\\n",
    "            / len(pred_entity_set)\n",
    "    else:\n",
    "        precision = 0\n",
    "    if precision > 0 and recall > 0:\n",
    "        f1 = 2 * precision * recall / (precision + recall)\n",
    "    else:\n",
    "        f1 = 0\n",
    "    return precision, recall, f1\n",
    "\n",
    "train_P = hmm.decode(train_X)\n",
    "p, r, f = compute_metric(train_Y, train_P)\n",
    "print(f'precision = {p}, recall = {r}, f1 = {f}')\n",
    "test_P = hmm.decode(test_X)\n",
    "p, r, f = compute_metric(test_Y, test_P)\n",
    "print(f'precision = {p}, recall = {r}, f1 = {f}')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3abe8397",
   "metadata": {},
   "source": [
    "\n",
    "\n",
    "下面展示基于条件随机场的序列标注模型。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "787f8ad1",
   "metadata": {},
   "outputs": [],
   "source": [
    "\"\"\"\n",
    "代码修改自GitHub项目kmkurn/pytorch-crf\n",
    "（Copyright (c) 2019, Kemal Kurniawan, MIT License（见附录））\n",
    "\"\"\"\n",
    "import torch\n",
    "from torch import nn\n",
    "\n",
    "class CRFLayer(nn.Module):\n",
    "    def __init__(self, n_tags, n_features):\n",
    "        super().__init__()\n",
    "        self.n_tags = n_tags\n",
    "        self.n_features = n_features\n",
    "        # 定义模型参数\n",
    "        self.transitions = nn.Parameter(torch.empty(\\\n",
    "            n_tags, n_tags))\n",
    "        self.emission_weight = nn.Parameter(torch.empty(\\\n",
    "            n_features, n_tags))\n",
    "        self.start_transitions = nn.Parameter(torch.empty(n_tags))\n",
    "        self.end_transitions = nn.Parameter(torch.empty(n_tags))\n",
    "        self.reset_parameters()\n",
    "        \n",
    "    # 使用（-0.1,0.1）之间的均匀分布初始化参数\n",
    "    def reset_parameters(self):\n",
    "        nn.init.uniform_(self.transitions, -0.1, 0.1)\n",
    "        nn.init.uniform_(self.emission_weight, -0.1, 0.1)\n",
    "        nn.init.uniform_(self.start_transitions, -0.1, 0.1)\n",
    "        nn.init.uniform_(self.end_transitions, -0.1, 0.1)\n",
    "    \n",
    "    # 使用动态规划计算得分\n",
    "    def compute_score(self, emissions, tags, masks):\n",
    "        seq_len, batch_size, n_tags = emissions.shape\n",
    "        score = self.start_transitions[tags[0]] + \\\n",
    "            emissions[0, torch.arange(batch_size), tags[0]]\n",
    "        \n",
    "        for i in range(1, seq_len):\n",
    "            score += self.transitions[tags[i-1], tags[i]] * masks[i]\n",
    "            score += emissions[i, torch.arange(batch_size),\\\n",
    "                tags[i]] * masks[i]\n",
    "        \n",
    "        seq_ends = masks.long().sum(dim=0) - 1\n",
    "        last_tags = tags[seq_ends, torch.arange(batch_size)]\n",
    "        score += self.end_transitions[last_tags]\n",
    "        return score\n",
    "    \n",
    "    # 计算配分函数\n",
    "    def computer_normalizer(self, emissions, masks):\n",
    "        seq_len, batch_size, n_tags = emissions.shape\n",
    "        # batch_size * n_tags, [起始分数 + y_0为某标签的发射分数 ...]\n",
    "        score = self.start_transitions + emissions[0]\n",
    "        \n",
    "        for i in range(1, seq_len):\n",
    "            # batch_size * n_tags * 1 [y_{i-1}为某tag的总分]\n",
    "            broadcast_score = score.unsqueeze(2)\n",
    "            # batch_size * 1 * n_tags [y_i为某标签的发射分数]\n",
    "            broadcast_emissions = emissions[i].unsqueeze(1)\n",
    "            # batch_size * n_tags * n_tags [任意y_{i-1}到y_i的总分]\n",
    "            next_score = broadcast_score + self.transitions +\\\n",
    "                broadcast_emissions\n",
    "            # batch_size * n_tags [对y_{i-1}求和]\n",
    "            next_score = torch.logsumexp(next_score, dim=1)\n",
    "            # masks为True则更新，否则保留\n",
    "            score = torch.where(masks[i].unsqueeze(1), next_score, score)\n",
    "            \n",
    "        score += self.end_transitions\n",
    "        return torch.logsumexp(score, dim=1)\n",
    "    \n",
    "    def forward(self, features, tags, masks):\n",
    "        \"\"\"\n",
    "        features: seq_len * batch_size * n_features\n",
    "        tags/masks: seq_len * batch_size\n",
    "        \"\"\"\n",
    "        _, batch_size, _ = features.size()\n",
    "        emissions = torch.matmul(features, self.emission_weight)\n",
    "        masks = masks.to(torch.bool)\n",
    "        \n",
    "        score = self.compute_score(emissions, tags, masks)\n",
    "        partition = self.computer_normalizer(emissions, masks)\n",
    "        \n",
    "        likelihood = score - partition\n",
    "        return likelihood.sum() / batch_size\n",
    "    \n",
    "    def decode(self, features, masks):\n",
    "        # 与computer_normalizer类似，sum变为max\n",
    "        emissions = torch.matmul(features, self.emission_weight)\n",
    "        masks = masks.to(torch.bool)\n",
    "        \n",
    "        seq_len, batch_size, n_tags = emissions.shape\n",
    "        score = self.start_transitions + emissions[0]\n",
    "        history = []\n",
    "        \n",
    "        for i in range(1, seq_len):\n",
    "            broadcast_score = score.unsqueeze(2)\n",
    "            broadcast_emission = emissions[i].unsqueeze(1)\n",
    "            \n",
    "            next_score = broadcast_score + self.transitions +\\\n",
    "                broadcast_emission\n",
    "            next_score, indices = next_score.max(dim=1)\n",
    "            \n",
    "            score = torch.where(masks[i].unsqueeze(1), next_score,\\\n",
    "                score)\n",
    "            history.append(indices)\n",
    "            \n",
    "        score += self.end_transitions\n",
    "        seq_ends = masks.long().sum(dim=0) - 1\n",
    "        best_tags_list = []\n",
    "        \n",
    "        for idx in range(batch_size):\n",
    "            _, best_last_tag = score[idx].max(dim=0)\n",
    "            best_tags = [best_last_tag.item()]\n",
    "            \n",
    "            for hist in reversed(history[:seq_ends[idx]]):\n",
    "                best_last_tag = hist[idx][best_tags[-1]]\n",
    "                best_tags.append(best_last_tag.item())\n",
    "                \n",
    "            best_tags.reverse()\n",
    "            best_tags_list.append(best_tags)\n",
    "            \n",
    "        return best_tags_list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "69f4af96",
   "metadata": {},
   "outputs": [],
   "source": [
    "class CRF(nn.Module):\n",
    "    def __init__(self, vocab_size, hidden_size, n_tags):\n",
    "        super().__init__()\n",
    "        self.embedding = nn.Embedding(vocab_size, hidden_size)\n",
    "        self.crf = CRFLayer(n_tags, hidden_size)\n",
    "        \n",
    "    def forward(self, input_ids, masks, labels):\n",
    "        \"\"\"\n",
    "        input_ids/masks/labels: batch_size * seq_len\n",
    "        \"\"\"\n",
    "        # 将输入序列转化为词嵌入序列\n",
    "        # batch_size * seq_len * embed_size\n",
    "        embed = self.embedding(input_ids)\n",
    "        embed = torch.transpose(embed, 0, 1)\n",
    "        masks = torch.transpose(masks, 0, 1)\n",
    "        labels = torch.transpose(labels, 0, 1)\n",
    "        # 隐状态和标签、掩码输入条件随机场计算损失\n",
    "        llh = self.crf(embed, labels, masks)\n",
    "        return -llh\n",
    "    \n",
    "    def decode(self, input_ids, masks):\n",
    "        embed = self.embedding(input_ids)\n",
    "        embed = torch.transpose(embed, 0, 1)\n",
    "        masks = torch.transpose(masks, 0, 1)\n",
    "        # 调用CRFLayer进行解码\n",
    "        return self.crf.decode(embed, masks)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "178948b4",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "epoch-19, loss=40.08: 100%|█| 20/20 [01:01<00:00,  3.07s/it]\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAkQAAAGwCAYAAABIC3rIAAAAOnRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjEwLjEsIGh0dHBzOi8vbWF0cGxvdGxpYi5vcmcvc2/+5QAAAAlwSFlzAAAPYQAAD2EBqD+naQAAWIJJREFUeJzt3Xd4VFX+x/H3TJJJ7yEJgQQiNSBNQAywljVKExtrRUUXZXVBRRTLbxfWjqLrKlhYXdeyi3UFC4iCVIEYIBC6FAlJgBRIL6TO/f0RMjISIMAkM0k+r+eZ52HuPXPne+Ax+XjuOeeaDMMwEBEREWnFzM4uQERERMTZFIhERESk1VMgEhERkVZPgUhERERaPQUiERERafUUiERERKTVUyASERGRVs/d2QU0B1arlUOHDuHv74/JZHJ2OSIiItIAhmFQXFxMVFQUZvOpx4AUiBrg0KFDREdHO7sMEREROQsZGRm0b9/+lG0UiBrA398fqP0LDQgIcHI1IiIi0hBFRUVER0fbfo+figJRA9TdJgsICFAgEhERaWYaMt1Fk6pFRESk1VMgEhERkVZPgUhERERaPc0hEhERAWpqaqiqqnJ2GXKGLBbLaZfUN4QCkYiItGqGYZCVlUVBQYGzS5GzYDabiY2NxWKxnNN1FIhERKRVqwtD4eHh+Pj4aAPeZqRu4+TMzExiYmLO6d9OgUhERFqtmpoaWxgKDQ11djlyFtq0acOhQ4eorq7Gw8PjrK+jSdUiItJq1c0Z8vHxcXIlcrbqbpXV1NSc03UUiEREpNXTbbLmy1H/dgpEIiIi0uopEImIiEirp0AkIiLSynXs2JFXX33V6ddwJq0ycyLDMMgvqyK3pIIuEad/Eq+IiAjApZdeSt++fR0WQNavX4+vr69DrtVcKRA50b4jpVz+95X4WtzY9tQwTeoTERGHMQyDmpoa3N1P/6u+TZs2TVCRa9MtMyeKCvQGoLSyhqLyaidXIyIiUBskyiqrm/xlGEaD6rvzzjtZuXIlr732GiaTCZPJxP79+1mxYgUmk4lFixbRv39/PD09Wb16Nb/88gvXXHMNERER+Pn5MXDgQH744Qe7a/72dpfJZOJf//oX1113HT4+PnTp0oWvv/76jP4e09PTueaaa/Dz8yMgIIAbb7yR7Oxs2/nNmzdz2WWX4e/vT0BAAP3792fDhg0ApKWlMXr0aIKDg/H19aVnz558++23Z/T9Z8qpI0SrVq3ipZdeIjk5mczMTObPn8+1114L1O4N8de//pVvv/2Wffv2ERgYSEJCAi+88AJRUVG2a+Tl5XH//ffzzTffYDabGTNmDK+99hp+fn62Nlu2bGHixImsX7+eNm3acP/99/Poo482dXdP4G1xI9TXQm5pJYcKjhLoffYbSomIiGMcraqhx/Tvm/x7dzw9DB/L6X8tv/baa+zevZvzzz+fp59+Gqgd4dm/fz8Ajz/+OC+//DLnnXcewcHBZGRkMHLkSJ577jk8PT358MMPGT16NLt27SImJuak3/PUU08xc+ZMXnrpJWbPns3YsWNJS0sjJCTktDVarVZbGFq5ciXV1dVMnDiRm266iRUrVgAwduxY+vXrx1tvvYWbmxspKSm2jRUnTpxIZWUlq1atwtfXlx07dtj9Xm8MTg1EpaWl9OnThz/+8Y9cf/31dufKysrYuHEj06ZNo0+fPuTn5/Pggw9y9dVX2xIk1P6FZmZmsmTJEqqqqrjrrruYMGECH330EQBFRUVceeWVJCQkMGfOHLZu3cof//hHgoKCmDBhQpP2tz5RQd62QBTXNsDZ5YiIiIsLDAzEYrHg4+NDZGTkCeeffvpprrjiCtv7kJAQ+vTpY3v/zDPPMH/+fL7++msmTZp00u+58847ueWWWwB4/vnnmTVrFuvWrWP48OGnrXHp0qVs3bqV1NRUoqOjAfjwww/p2bMn69evZ+DAgaSnpzN16lS6d+8OQJcuXWyfT09PZ8yYMfTq1QuA884777Tfea6cGohGjBjBiBEj6j0XGBjIkiVL7I69/vrrXHjhhaSnpxMTE8POnTv57rvvWL9+PQMGDABg9uzZjBw5kpdffpmoqCjmzp1LZWUl//73v7FYLPTs2ZOUlBReeeUVFwlEXmw9WMihgqPOLkVERABvDzd2PD3MKd/rCHW/D+uUlJTw5JNPsnDhQjIzM6murubo0aOkp6ef8jq9e/e2/dnX15eAgABycnIaVMPOnTuJjo62hSGAHj16EBQUxM6dOxk4cCBTpkzh7rvv5j//+Q8JCQnccMMNdOrUCYAHHniA++67j8WLF5OQkMCYMWPs6mkMzWoOUWFhISaTiaCgIAASExMJCgqy+8dPSEjAbDaTlJRka3PxxRfbPQV32LBh7Nq1i/z8/Hq/p6KigqKiIrtXY4kKqp1HdECBSETEJZhMJnws7k3+ctTCmt+uFnvkkUeYP38+zz//PD/++CMpKSn06tWLysrKU17nt88FM5lMWK1Wh9QI8OSTT7J9+3ZGjRrFsmXL6NGjB/Pnzwfg7rvvZt++fdx+++1s3bqVAQMGMHv2bId9d32aTSAqLy/nscce45ZbbiEgoPbWUlZWFuHh4Xbt3N3dCQkJISsry9YmIiLCrk3d+7o2vzVjxgwCAwNtr+MTrqO1OxaIDhWUN9p3iIhIy2KxWBr87K41a9Zw5513ct1119GrVy8iIyNt840aS1xcHBkZGWRkZNiO7dixg4KCAnr06GE71rVrVx566CEWL17M9ddfz3vvvWc7Fx0dzb333su8efN4+OGHeeeddxq15mYRiKqqqrjxxhsxDIO33nqr0b/viSeeoLCw0PY6/h/U0aJsgUgjRCIi0jAdO3YkKSmJ/fv3c+TIkVOO3HTp0oV58+aRkpLC5s2bufXWWx060lOfhIQEevXqxdixY9m4cSPr1q3jjjvu4JJLLmHAgAEcPXqUSZMmsWLFCtLS0lizZg3r168nLi4OgMmTJ/P999+TmprKxo0bWb58ue1cY3H5QFQXhtLS0liyZIltdAggMjLyhPuZ1dXV5OXl2SaaRUZG2i3zA2zv65uMBuDp6UlAQIDdq7G0UyASEZEz9Mgjj+Dm5kaPHj1o06bNKecDvfLKKwQHBzN48GBGjx7NsGHDuOCCCxq1PpPJxFdffUVwcDAXX3wxCQkJnHfeeXz66acAuLm5kZubyx133EHXrl258cYbGTFiBE899RRQ++T6iRMnEhcXx/Dhw+natStvvvlmo9bs0hsz1oWhPXv2sHz5ckJDQ+3Ox8fHU1BQQHJyMv379wdg2bJlWK1WBg0aZGvzl7/8haqqKtv90CVLltCtWzeCg4ObtkP1qBshyi4qp6rGioeby2dUERFxsq5du5KYmGh3rGPHjvXuZdSxY0eWLVtmd2zixIl27397C62+6xQUFJyypt9eIyYmhq+++qrethaLhY8//vik12rs+UL1cepv35KSElJSUkhJSQEgNTWVlJQU0tPTqaqq4g9/+AMbNmxg7ty51NTUkJWVRVZWlm0iWF1yvOeee1i3bh1r1qxh0qRJ3Hzzzba9im699VYsFgvjx49n+/btfPrpp7z22mtMmTLFWd22E+prweJuxmrUhiIRERFpek4NRBs2bKBfv37069cPgClTptCvXz+mT5/OwYMH+frrrzlw4AB9+/albdu2ttfatWtt15g7dy7du3fn8ssvZ+TIkQwdOpS3337bdj4wMJDFixeTmppK//79efjhh5k+fbpLLLkHMJtNRAV6AXAwX7fNREREnMGpt8wuvfTSU25V3pBtzENCQmybMJ5M7969+fHHH8+4vqYSFeTN/twyDhUqEImIiDiDJqy4gCgtvRcRcaqGPkdMXI+j/u0UiFxAXSA6qJVmIiJNqm6xTVlZmZMrkbNVN6/Yze3cdvp26VVmrUV7Lb0XEXEKNzc3goKCbFu4+Pj4OGzHaGl8VquVw4cP4+Pjg7v7uUUaBSIXoM0ZRUScp25PuoY+p0tci9lsJiYm5pyDrAKRC4gK+nWVmWEY+r8TEZEmZDKZaNu2LeHh4VRVVTm7HDlDFosFs/ncZwApELmAuhGi0soaio5WE+jjcZpPiIiIo7m5uZ3zPBRpvjSp2gV4ebgR6msBNLFaRETEGRSIXITmEYmIiDiPApGLsD3kVZszioiINDkFIhehvYhEREScR4HIRdStNNNu1SIiIk1PgchF1N0yO5iv3VJFRESamgKRi9DzzERERJxHgchFtD12yyynuJwaqx4yKCIi0pQUiFxEqK8nZhNYDcgrrXR2OSIiIq2KApGLcDObCDm2OePh4gonVyMiItK6KBC5kDA/TwCOlCgQiYiINCUFIhfSxr82EGmESEREpGkpELmQNsdGiA5rhEhERKRJKRC5kLBjI0RHNEIkIiLSpBSIXIhGiERERJxDgciF1M0h0qRqERGRpqVA5ELqVplpUrWIiEjTUiByIb+OEGljRhERkaakQORCwvxqN2bMK62kqsbq5GpERERaDwUiFxLsY8HNbAL0+A4REZGmpEDkQsxmE6F6fIeIiEiTUyByMbbdqrXSTEREpMkoELkYrTQTERFpegpELkbPMxMREWl6CkQuRk+8FxERaXoKRC5GI0QiIiJNT4HIxdTtRaQRIhERkaajQORiNEIkIiLS9BSIXEwbPz2+Q0REpKkpELmYuhGiwqNVVFTXOLkaERGR1kGByMUEenvg4Vb7+I5cjRKJiIg0CQUiF2MymbQ5o4iISBNTIHJB2otIRESkaSkQuSCtNBMREWlaCkQuqG4vIgUiERGRpqFA5ILqRoh0y0xERKRpKBC5oLq9iA4rEImIiDQJBSIXFFY3QlSsZfciIiJNQYHIBWmESEREpGkpELmgX0eIFIhERESaglMD0apVqxg9ejRRUVGYTCa+/PJLu/OGYTB9+nTatm2Lt7c3CQkJ7Nmzx65NXl4eY8eOJSAggKCgIMaPH09JSYldmy1btvC73/0OLy8voqOjmTlzZmN37ZzUTaourqimvEqP7xAREWlsTg1EpaWl9OnThzfeeKPe8zNnzmTWrFnMmTOHpKQkfH19GTZsGOXl5bY2Y8eOZfv27SxZsoQFCxawatUqJkyYYDtfVFTElVdeSYcOHUhOTuall17iySef5O233270/p0tf093LO61/zRaei8iItIEDBcBGPPnz7e9t1qtRmRkpPHSSy/ZjhUUFBienp7Gxx9/bBiGYezYscMAjPXr19vaLFq0yDCZTMbBgwcNwzCMN9980wgODjYqKipsbR577DGjW7duDa6tsLDQAIzCwsKz7d4ZGzxjqdHhsQVGclpek32niIhIS3Imv79ddg5RamoqWVlZJCQk2I4FBgYyaNAgEhMTAUhMTCQoKIgBAwbY2iQkJGA2m0lKSrK1ufjii7FYLLY2w4YNY9euXeTn59f73RUVFRQVFdm9mprmEYmIiDQdlw1EWVlZAERERNgdj4iIsJ3LysoiPDzc7ry7uzshISF2beq7xvHf8VszZswgMDDQ9oqOjj73Dp0hrTQTERFpOi4biJzpiSeeoLCw0PbKyMho8hra+NeOaGkvIhERkcbnsoEoMjISgOzsbLvj2dnZtnORkZHk5OTYna+uriYvL8+uTX3XOP47fsvT05OAgAC7V1P7dYSo/DQtRURE5Fy5bCCKjY0lMjKSpUuX2o4VFRWRlJREfHw8APHx8RQUFJCcnGxrs2zZMqxWK4MGDbK1WbVqFVVVVbY2S5YsoVu3bgQHBzdRb85cmJ54LyIi0mScGohKSkpISUkhJSUFqJ1InZKSQnp6OiaTicmTJ/Pss8/y9ddfs3XrVu644w6ioqK49tprAYiLi2P48OHcc889rFu3jjVr1jBp0iRuvvlmoqKiALj11luxWCyMHz+e7du38+mnn/Laa68xZcoUJ/W6YepGiI6U6JaZiIhIY3N35pdv2LCByy67zPa+LqSMGzeO999/n0cffZTS0lImTJhAQUEBQ4cO5bvvvsPLy8v2mblz5zJp0iQuv/xyzGYzY8aMYdasWbbzgYGBLF68mIkTJ9K/f3/CwsKYPn263V5FrkgjRCIiIk3HZBiG4ewiXF1RURGBgYEUFhY22Xyi/UdKufTlFfhY3Njx9PAm+U4REZGW5Ex+f7vsHKLWrm6EqKyyhtKKaidXIyIi0rIpELkoX4sb3h5uABzRXkQiIiKNSoHIRZlMJsKO7UWkeUQiIiKNS4HIhf260kyBSEREpDEpELmwNlppJiIi0iQUiFxYmG23au1FJCIi0pgUiFyYRohERESahgKRC7ONECkQiYiINCoFIhdWN0KkSdUiIiKNS4HIhWmESEREpGkoELmw8ONGiPSEFRERkcajQOTC6kaIKqqtFOvxHSIiIo1GgciFeVvc8PN0B+CIbpuJiIg0GgUiFxfmp8d3iIiINDYFIhf360ozbc4oIiLSWBSIXNyvK83KnVyJiIhIy6VA5OI0QiQiItL4FIhcnPYiEhERaXwKRC7O9jwz7VYtIiLSaBSIXFzdCJEe3yEiItJ4FIhcnJ54LyIi0vgUiFxc3T5EuSWVenyHiIhII1EgcnGhvrUjRJU1Vkr0+A4REZFGoUDk4rwtbnh7uAGQV6ql9yIiIo1BgagZCPE9dttMgUhERKRRKBA1A3WBKE+bM4qIiDQKBaJmwBaINEIkIiLSKBSImoHQukBUpkAkIiLSGBSImgGNEImIiDQuBaJmIOS4vYhERETE8RSImgHbLbNS7VYtIiLSGBSImoFgH90yExERaUwKRM1AqJ8mVYuIiDQmBaJmIOTY4zu0D5GIiEjjUCBqBupWmZVW1lBeVePkakRERFoeBaJmIMDLHQ83E6B5RCIiIo1BgagZMJlMmlgtIiLSiBSImgk94FVERKTxKBA1E3WBKF+BSERExOEUiJoJjRCJiIg0HgWiZkK7VYuIiDQeBaJmwrYXkUaIREREHE6BqJnQA15FREQajwJRM/HrLTMFIhEREUdTIGombPsQ6XlmIiIiDqdA1EzYHvCqESIRERGHUyBqJuqW3ReUVVFdY3VyNSIiIi2LSweimpoapk2bRmxsLN7e3nTq1IlnnnkGwzBsbQzDYPr06bRt2xZvb28SEhLYs2eP3XXy8vIYO3YsAQEBBAUFMX78eEpKSpq6O+ck2MeCqfZxZuSXVTm3GBERkRbGpQPRiy++yFtvvcXrr7/Ozp07efHFF5k5cyazZ8+2tZk5cyazZs1izpw5JCUl4evry7BhwygvL7e1GTt2LNu3b2fJkiUsWLCAVatWMWHCBGd06ay5mU0EeXsAum0mIiLiaO7OLuBU1q5dyzXXXMOoUaMA6NixIx9//DHr1q0DakeHXn31Vf76179yzTXXAPDhhx8SERHBl19+yc0338zOnTv57rvvWL9+PQMGDABg9uzZjBw5kpdffpmoqKgTvreiooKKil83QCwqKmrsrjZIiK+F/LIqBSIREREHc+kRosGDB7N06VJ2794NwObNm1m9ejUjRowAIDU1laysLBISEmyfCQwMZNCgQSQmJgKQmJhIUFCQLQwBJCQkYDabSUpKqvd7Z8yYQWBgoO0VHR3dWF08I6HanFFERKRRuPQI0eOPP05RURHdu3fHzc2NmpoannvuOcaOHQtAVlYWABEREXafi4iIsJ3LysoiPDzc7ry7uzshISG2Nr/1xBNPMGXKFNv7oqIilwhFwb51t8z0+A4RERFHculA9NlnnzF37lw++ugjevbsSUpKCpMnTyYqKopx48Y12vd6enri6enZaNc/W3WP79ADXkVERBzLpQPR1KlTefzxx7n55psB6NWrF2lpacyYMYNx48YRGRkJQHZ2Nm3btrV9Ljs7m759+wIQGRlJTk6O3XWrq6vJy8uzfb650G7VIiIijcOl5xCVlZVhNtuX6ObmhtVauw9PbGwskZGRLF261Ha+qKiIpKQk4uPjAYiPj6egoIDk5GRbm2XLlmG1Whk0aFAT9MJx6vYi0giRiIiIY7n0CNHo0aN57rnniImJoWfPnmzatIlXXnmFP/7xjwCYTCYmT57Ms88+S5cuXYiNjWXatGlERUVx7bXXAhAXF8fw4cO55557mDNnDlVVVUyaNImbb7653hVmrqxut+p8BSIRERGHculANHv2bKZNm8af//xncnJyiIqK4k9/+hPTp0+3tXn00UcpLS1lwoQJFBQUMHToUL777ju8vLxsbebOncukSZO4/PLLMZvNjBkzhlmzZjmjS+ckRLfMREREGoXJOH7bZ6lXUVERgYGBFBYWEhAQ4LQ6th8qZNSs1bTx92T9XxJO/wEREZFW7Ex+f7v0HCKxV7cPUX5pJVarcqyIiIijKBA1I3W3zKqtBkXlep6ZiIiIoygQNSMWdzMBXrXTvo6UaB6RiIiIoygQNTNhfsc2ZyzRbtUiIiKOokDUzNQtvddeRCIiIo6jQNTM1E2s1giRiIiI4ygQNTN1I0SaQyQiIuI4CkTNTGjdHCI98V5ERMRhFIiambC6EaJijRCJiIg4igJRM2ObQ6QRIhEREYdRIGpmbKvMNIdIRETEYRSImhnbLTOtMhMREXEYBaJmpu6WWVF5NZXVVidXIyIi0jIoEDUzgd4euJlNAORpc0YRERGHUCBqZsxmk+0hr7ptJiIi4hgKRM1QqK8e3yEiIuJICkTNkB7wKiIi4lgKRM2Qlt6LiIg4lgJRM1S30uyINmcUERFxCAWiZkgjRCIiIo6lQNQMhdkCkUaIREREHEGBqBn69XlmGiESERFxBAWiZki3zERERBxLgagZqlt2f6SkAsMwnFyNiIhI86dA1AzVjRBVVFspraxxcjUiIiLNnwJRM+Rjccfbww3QxGoRERFHUCBqpupGiY5oHpGIiMg5UyBqpkL1+A4RERGHUSBqpsL0gFcRERGHUSBqpkK1OaOIiIjDKBA1U6G2pfcaIRIRETlXCkTNVKhumYmIiDjMWQWiDz74gIULF9reP/roowQFBTF48GDS0tIcVpycXJgmVYuIiDjMWQWi559/Hm9vbwASExN54403mDlzJmFhYTz00EMOLVDqp8d3iIiIOI772XwoIyODzp07A/Dll18yZswYJkyYwJAhQ7j00ksdWZ+cRN0DXo9ohEhEROScndUIkZ+fH7m5uQAsXryYK664AgAvLy+OHj3quOrkpNr41waivLJKqmusTq5GRESkeTurEaIrrriCu+++m379+rF7925GjhwJwPbt2+nYsaMj65OTCPG1YDaB1aidWB0R4OXskkRERJqtsxoheuONN4iPj+fw4cN88cUXhIaGApCcnMwtt9zi0AKlfm5mk23p/eFi3TYTERE5F2c1QhQUFMTrr79+wvGnnnrqnAuShgv39+RwcQU5xeVAoLPLERERabbOaoTou+++Y/Xq1bb3b7zxBn379uXWW28lPz/fYcXJqdXNI9IIkYiIyLk5q0A0depUioqKANi6dSsPP/wwI0eOJDU1lSlTpji0QDm58GOBKKdIgUhERORcnNUts9TUVHr06AHAF198wVVXXcXzzz/Pxo0bbROspfHZRoi09F5EROScnNUIkcVioaysDIAffviBK6+8EoCQkBDbyJE0vnD/2pVlumUmIiJybs5qhGjo0KFMmTKFIUOGsG7dOj799FMAdu/eTfv27R1aoJxc3QhRjgKRiIjIOTmrEaLXX38dd3d3/ve///HWW2/Rrl07ABYtWsTw4cMdWqCcXLgmVYuIiDjEWY0QxcTEsGDBghOO/+Mf/zjngqThfh0hKscwDEwmk5MrEhERaZ7OaoQIoKamhi+++IJnn32WZ599lvnz51NTU+PI2gA4ePAgt912G6GhoXh7e9OrVy82bNhgO28YBtOnT6dt27Z4e3uTkJDAnj177K6Rl5fH2LFjCQgIICgoiPHjx1NSUuLwWptaXSAqr7JSUlHt5GpERESar7MKRHv37iUuLo477riDefPmMW/ePG677TZ69uzJL7/84rDi8vPzGTJkCB4eHixatIgdO3bw97//neDgYFubmTNnMmvWLObMmUNSUhK+vr4MGzaM8vJyW5uxY8eyfft2lixZwoIFC1i1ahUTJkxwWJ3O4mNxx8+zdpBP84hERETOnskwDONMPzRy5EgMw2Du3LmEhIQAkJuby2233YbZbGbhwoUOKe7xxx9nzZo1/Pjjj/WeNwyDqKgoHn74YR555BEACgsLiYiI4P333+fmm29m586d9OjRg/Xr1zNgwACgdmPJkSNHcuDAAaKiok64bkVFBRUVvwaMoqIioqOjKSwsJCAgwCF9c5TLXl5B6pFSPplwERedF+rsckRERFxGUVERgYGBDfr9fVYjRCtXrmTmzJm2MAQQGhrKCy+8wMqVK8/mkvX6+uuvGTBgADfccAPh4eH069ePd955x3Y+NTWVrKwsEhISbMcCAwMZNGgQiYmJACQmJhIUFGQLQwAJCQmYzWaSkpLq/d4ZM2YQGBhoe0VHRzusT46m3apFRETO3VkFIk9PT4qLi084XlJSgsViOeei6uzbt4+33nqLLl268P3333PffffxwAMP8MEHHwCQlZUFQEREhN3nIiIibOeysrIIDw+3O+/u7k5ISIitzW898cQTFBYW2l4ZGRkO65Ojaem9iIjIuTurVWZXXXUVEyZM4N133+XCCy8EICkpiXvvvZerr77aYcVZrVYGDBjA888/D0C/fv3Ytm0bc+bMYdy4cQ77nt/y9PTE09Oz0a7vSFp6LyIicu7OaoRo1qxZdOrUifj4eLy8vPDy8mLw4MF07tyZV1991WHFtW3b1vaIkDpxcXGkp6cDEBkZCUB2drZdm+zsbNu5yMhIcnJy7M5XV1eTl5dna9OcHb/0XkRERM7OWY0QBQUF8dVXX7F371527twJ1AaVzp07O7S4IUOGsGvXLrtju3fvpkOHDgDExsYSGRnJ0qVL6du3L1A7gSopKYn77rsPgPj4eAoKCkhOTqZ///4ALFu2DKvVyqBBgxxarzPo8R0iIiLnrsGB6HRPsV++fLntz6+88srZV3Schx56iMGDB/P8889z4403sm7dOt5++23efvttAEwmE5MnT+bZZ5+lS5cuxMbGMm3aNKKiorj22muB2qA2fPhw7rnnHubMmUNVVRWTJk3i5ptvrneFWXOjSdUiIiLnrsGBaNOmTQ1q58jdkgcOHMj8+fN54oknePrpp4mNjeXVV19l7NixtjaPPvoopaWlTJgwgYKCAoYOHcp3332Hl5eXrc3cuXOZNGkSl19+OWazmTFjxjBr1iyH1elMbfwUiERERM7VWe1D1NqcyT4GTe1ISQUDnv0Bkwl2PzsCD7ez3nxcRESkRWn0fYjEdYT4WHAzmzAMyC2pdHY5IiIizZICUTNnNpsI86vd+0m3zURERM6OAlELYJtYXaKl9yIiImdDgagFqFt6n1OkESIREZGzoUDUAmilmYiIyLlRIGoB9DwzERGRc6NA1AKEB2iESERE5FwoELUAdbfM9DwzERGRs6NA1ALUjRBla1K1iIjIWVEgagFiw/wAOFhwlLxSbc4oIiJyphSIWoAQXwtdwmtD0fr9eU6uRkREpPlRIGohLowNASBpnwKRiIjImVIgaiEGnRcKwLr9uU6uREREpPlRIGohBh0bIdpxqIii8ionVyMiItK8KBC1EBEBXnQM9cFqwAbNIxIRETkjCkQtyKDY2ttmSakKRCIiImdCgagF0cRqERGRs6NA1IIMOq82EG07WEhpRbWTqxEREWk+FIhakPbBPrQL8qbaarAxPd/Z5YiIiDQbCkQtTN1qs3WaRyQiItJgCkQtjOYRiYiInDkFohamf4dgALYfKsQwDCdXIyIi0jwoELUwMaE+mExQWlnDkRI96FVERKQhFIhaGE93N6ICvQFIyy11cjUiIiLNgwJRCxQb5gvA/twyJ1ciIiLSPCgQtUAdQn0A2H9EI0QiIiINoUDUAnUMrRshUiASERFpCAWiFqhuhChNt8xEREQaRIGoBbLNITpSqqX3IiIiDaBA1AJFh9QuvS+uqCavVEvvRURETkeBqAXy8nCjbYAXoJVmIiIiDaFA1EJ1ODaxWnsRiYiInJ4CUQvVMUxL70VERBpKgaiF+nXpvW6ZiYiInI4CUQulW2YiIiINp0DUQtXdMkvV0nsREZHTUiBqoTqE1I4QFZVXU1BW5eRqREREXJsCUQvlbXEj0rb0XrfNRERETkWBqAXTIzxEREQaRoGoBatbaZaqpfciIiKnpEDUgnUIqxshUiASERE5FQWiFiy2boRIt8xEREROSYGoBesa6Q/AjkOF5BSXO7kaERER16VA1IJ1auNHv5ggqmoMPl2X4exyREREXJYCUQt3R3wHAD5al051jdXJ1YiIiLgmBaIWbmSvtoT4WsgsLOeHndnOLkdERMQlNatA9MILL2AymZg8ebLtWHl5ORMnTiQ0NBQ/Pz/GjBlDdrb9L/709HRGjRqFj48P4eHhTJ06lerq6iau3jk83d24aWA0AB8mpjm5GhEREdfUbALR+vXr+ec//0nv3r3tjj/00EN88803fP7556xcuZJDhw5x/fXX287X1NQwatQoKisrWbt2LR988AHvv/8+06dPb+ouOM3YQTGYTbD2l1z25hQ7uxwRERGX0ywCUUlJCWPHjuWdd94hODjYdrywsJB3332XV155hd///vf079+f9957j7Vr1/LTTz8BsHjxYnbs2MF///tf+vbty4gRI3jmmWd44403qKysrPf7KioqKCoqsns1Z+2Dffh99wgA/vtTupOrERERcT3NIhBNnDiRUaNGkZCQYHc8OTmZqqoqu+Pdu3cnJiaGxMREABITE+nVqxcRERG2NsOGDaOoqIjt27fX+30zZswgMDDQ9oqOjm6EXjWt249Nrv4q5SA1VsPJ1YiIiLgWlw9En3zyCRs3bmTGjBknnMvKysJisRAUFGR3PCIigqysLFub48NQ3fm6c/V54oknKCwstL0yMpr/kvXBnULx93Qnv6yKbQcLnV2OiIiIS3HpQJSRkcGDDz7I3Llz8fLyarLv9fT0JCAgwO7V3Hm4mRncORSAlbsPO7kaERER1+LSgSg5OZmcnBwuuOAC3N3dcXd3Z+XKlcyaNQt3d3ciIiKorKykoKDA7nPZ2dlERkYCEBkZecKqs7r3dW1ai0u6hgOwSoFIRETEjksHossvv5ytW7eSkpJiew0YMICxY8fa/uzh4cHSpUttn9m1axfp6enEx8cDEB8fz9atW8nJybG1WbJkCQEBAfTo0aPJ++RMF3cNA2BTRgGFR6ucXI2IiIjrcHd2Aafi7+/P+eefb3fM19eX0NBQ2/Hx48czZcoUQkJCCAgI4P777yc+Pp6LLroIgCuvvJIePXpw++23M3PmTLKysvjrX//KxIkT8fT0bPI+OVP7YB/Oa+PLvsOlrN17hBG92jq7JBEREZfg0iNEDfGPf/yDq666ijFjxnDxxRcTGRnJvHnzbOfd3NxYsGABbm5uxMfHc9ttt3HHHXfw9NNPO7Fq57mkaxtA84hERESOZzIMQ2uwT6OoqIjAwEAKCwub/QTr5btyuOu99UQFerHm8d9jMpmcXZKIiEijOJPf381+hEjOzEWxoVjczRwqLGdvTomzyxEREXEJCkStjLfFjUGxIYBum4mIiNRRIGqF6uYRfbP5EOVVNU6uRkRExPkUiFqhYT0jsbib2XygkNv+lUR+af3PdBMREWktFIhaoegQHz6460L8vdzZkJbPmDlrycgrc3ZZIiIiTqNA1ErFdwrli/sGExXoxb7DpfzpP8lowaGIiLRWCkStWNcIf77482A83c3syCxiU0aBs0sSERFxCgWiVq5toDejju1Y/cm6dCdXIyIi4hwKRMItg2IA+GZzJsXlesaZiIi0PgpEwoAOwXQO9+NoVQ1fbz7k7HJERESanAKRYDKZuHlgNACfrMtwcjUiIiJNT4FIALj+gvZY3MxsPVjItoOFzi5HRESkSSkQCQAhvhau7BkBwMeaXC0iIq2MApHY3HpscvXnyQe0UaOIiLQqCkRiE39eKEM6h1JZbeX5b3c6uxwREZEmo0AkNiaTiWlX9cBsgkXbsvhpX66zSxIREWkSCkRip3tkgO3W2VPf7KDGqsd5iIhIy6dAJCeYckU3Arzc2ZlZxAdr9zu7HBERkUanQCQnCPG18GBCVwCeXrCD8e+vZ9/hEidXJSIi0ngUiKRe4+I78KeLz8PdbGLpzzkMe3UVHybud3ZZIiIijUKBSOrl7mbmiZFxfP/QxVzarQ1VNQbPLNhB6pHSk34mI6+M8qqaJqxSRETEMRSI5JQ6tfHjvTsHcknX2lD03ML6l+Nv2J/HxS8t5+HPNjdxhSIiIudOgUhOq3Y5fhzuZhM/7Mzmxz2HT2jzZcpBDAO+3ZZJeq42dRQRkeZFgUgapHO4P3fEdwTg6W92UF1jtZ0zDINlO3OO/RnmrktzRokiIiJnTYFIGuzBy7sQ7OPBnpwS5ib9+ryznZnFHCost73/fMMBzSUSEZFmRYFIGizQx4OHr+wGwOxle22hZ+nObAAu69aGtoFe5JVWsmhbptPqFBEROVMKRHJGbhoYTbsgb46UVPDFxgMALP259nbZlT0jueXC2l2u//tT+kmvISIi4moUiOSMeLiZuft3sQC8vWof2UXlbD5QAMDvu4dz88Bo3M0mktPy2ZlZ5MRKRUREGk6BSM7YTQOjCfbxIC23jMe+2IJhQK92gUQEeBEe4MWwnpEA/Ht1qpMrFRERaRgFIjljPhZ3xg3uCMCKXbVL8H/fPdx2/q4htec+Tz7Amr1Hmro8ERGRM6ZAJGdlXHxHvD3cbO8T4iJsfx7QMYTbLqqdSzT1880UlVc1eX0iIiJnQoFIzkqwr4WbL4wGINzfk55RAXbn/29kHB1CfThUWM7T3+xwRokiIiINpkAkZ23iZZ1JiAvniZHdMZtNdud8LO78/YY+mEzwv+QDLN6e5aQqRURETk+BSM5amJ8n/xo3kOv6ta/3/ICOIUy4+DwA/vb1do5WarNGERFxTQpE0qgeSuhKuyBvMgvL+eeqX2zH80srmfndz2w7WOjE6kRERGopEEmj8vJw44mR3QGYs/IXDhUcpaSimjvfW8ebK37hgY832T0XTURExBkUiKTRjerVlgs7hlBeZeW5hTuZ8OEGNh+oHRnad6SUrzcfcnKFIiLS2ikQSaMzmUxMH90DkwkWbs1k7S+5+FrcuP6CdgDMWrpHo0QiIuJUCkTSJM5vF8gN/WsnX1vczbwzbgDPXHM+Ib4W9ueW8WWKRolERMR5FIikyfzfyDjGDorhvTsHMrhTGL6e7rZVaLOXaZRIREScR4FImkyQj4XnruvFkM5htmN3xHcg1NdCWm4Z8zYddGJ1IiLSmikQiVP5WNz50yW1o0T/WLJbexWJiIhTKBCJ090R39G2V9E7P+5zdjkiItIKKRCJ03l5uPHYiNq9it5a8QvZReUntFm/P4/eT37PPR9uID23rKlLFBGRFk6BSFzC6N5tuSAmiKNVNbz0/a4Tzr+9ah9F5dUs2ZFNwj9W8vL3uyiv0u01ERFxDAUicQkmk4lpV/UA4IuNB+we6VFQVsmKXTkAXBATRGW1ldeX7+WOf6/TnCMREXEIlw5EM2bMYODAgfj7+xMeHs61117Lrl32owfl5eVMnDiR0NBQ/Pz8GDNmDNnZ2XZt0tPTGTVqFD4+PoSHhzN16lSqq6ubsivSAP1igrmmbxSGATMW7bQd/3ZrFlU1Bt0j/fnivsHMua0//p7urEvN497/JlNRrVAkIiLnxqUD0cqVK5k4cSI//fQTS5YsoaqqiiuvvJLS0lJbm4ceeohvvvmGzz//nJUrV3Lo0CGuv/562/mamhpGjRpFZWUla9eu5YMPPuD9999n+vTpzuiSnMbUYd3wcDOxZm8ua385AsCXKbXL8a/t1w6TycTw8yN5766BeHu4sXL3YR78OEV7GImIyDkxGYZhOLuIhjp8+DDh4eGsXLmSiy++mMLCQtq0acNHH33EH/7wBwB+/vln4uLiSExM5KKLLmLRokVcddVVHDp0iIiICADmzJnDY489xuHDh7FYLCd8T0VFBRUVFbb3RUVFREdHU1hYSEBAQNN0thWb/tU2PkxMo3+HYGbd0o8hLyzDZII1j/2eqCBvW7sf9xxm/PsbqKyxct+lnXhseHcnVi0iIq6mqKiIwMDABv3+dukRot8qLKydVxISEgJAcnIyVVVVJCQk2Np0796dmJgYEhMTAUhMTKRXr162MAQwbNgwioqK2L59e73fM2PGDAIDA22v6OjoxuqS1GPiZZ3xdDeTnJbP419sAeDCjiF2YQjgd13a8PKNfQD49+rUeleniYiINESzCURWq5XJkyczZMgQzj//fACysrKwWCwEBQXZtY2IiCArK8vW5vgwVHe+7lx9nnjiCQoLC22vjIwMB/dGTiUiwItxgzsC8OOe2ttm1/RtV2/b0b3b0r9DMBXVVmYv29NUJYqISAvTbALRxIkT2bZtG5988kmjf5enpycBAQF2L2laf7r4PHwtbgB4uJkY2Suy3nYmk4mpw7oB8Mm6DDLytEeRiIicuWYRiCZNmsSCBQtYvnw57du3tx2PjIyksrKSgoICu/bZ2dlERkba2vx21Vnd+7o24npC/TwZPzQWgIS4CIJ8TpzrVeei80L5XZcwqq0Gr/6gUSIRETlz7s4u4FQMw+D+++9n/vz5rFixgtjYWLvz/fv3x8PDg6VLlzJmzBgAdu3aRXp6OvHx8QDEx8fz3HPPkZOTQ3h4OABLliwhICCAHj16NG2H5Iw8cHkXOoX78bsubU7b9pEru/HjniPM33QALw8z6Xll5JZUcm2/KP44JBZ3t2aR/UVExElcepXZn//8Zz766CO++uorunXrZjseGBiIt3ftBNv77ruPb7/9lvfff5+AgADuv/9+ANauXQvULrvv27cvUVFRzJw5k6ysLG6//Xbuvvtunn/++QbVcSaz1MV5Jny4gcU7sk84fn67AJ6/rhdmk4mUjAL2HynljviOxIT6OKFKERFpKmfy+9ulA5HJZKr3+Hvvvcedd94J1G7M+PDDD/Pxxx9TUVHBsGHDePPNN+1uh6WlpXHfffexYsUKfH19GTduHC+88ALu7g0bIFMgah4yC4/y6pI9BPl4EBvmS3lVDa8s2U1R+YmbcLYL8mb+nwcTHuDlhEpFRKQptJhA5CoUiJqvnOJynvp6Bwu3ZuLv5U7f6CD255aSkXeUHm0D+PRPF+Hv5dGga63ec4QXvtvJC9f35vx2gY1cuYiInCsFIgdTIGr+Siqq8fFww2w2kZZbypi31nKkpJKhncN4544BeB9b0XYqo2evZuvBQq7v145Xburb+EWLiMg5abEbM4qcLT9Pd8zm2luwHUJ9+fedA/GxuLF67xEGPLuEBz7exOLtWdRY6///g11ZxWw99sDZlbsPYz1JOxERaZ4UiKRV6t0+iHfuGEC7IG9KK2v4evMhJvwnmfv+m0x51YkPi/1i4wHbn3NLK23hSEREWgYFImm1hnQOY/VjlzH/z4MZPzQWi7uZxTuyGf/Bekorfp2IXV1jZd7G2gfMhvnV7oe0fFeOU2oWEZHGoUAkrZrJZKJfTDDTrurB+8duo63Zm8tt7yZRUFYJwKo9hzlSUkGor4XJCV0BWLHrsDPLFhERB1MgEjlmcOcw5t49iEBvDzalF3D9W2tJyy3li+Ta0aGr+0aREFf7HLzNBwrILakAYOnObP7w1lp2ZhY5rXYRETk3CkQix+kXE8xnf4onKtCLfYdLue7NtSw5ttnjH/q3JzLQi7i2ARhG7cjRoYKjTP4khQ1p+fzf/K1o0aaISPOkQCTyG90i/fly4hB6tQskr7SSyhorcW0D6BlVu/fQpd1qHyWy7OfDTP3fZoqPzTfalF7Awq2ZTqtbRETOngKRSD3CA7z49E8XMaxn7S2ycfEdbOcu61b7TLwFWw6xZm8uXh5m/tC/9qHDL373MxXVJ65SExER1+bSD3cVcSYfiztzbutPdlEFEQGetuMXxATh7+VO8bFHgvxlZBxj+rdn1e7DZOQd5T+Jadz9u/OcVbaIiJwFjRCJnILJZCIy0MvuuXrubmYu7lp72+x3XcK47aIO+FjceeTK2gcQz1q6h+U/55BTXH7C9fJKK1mXmsfyn3OorLbanauqsZJVeOJnRESk8enRHQ2gR3fIbx3IL2PexoPcflEHgn1r9yaqsRqMmvUjP2cV29oF+Xjg6W7GbDJRXlVDflmV7VzHUB/+MqoHl3Vrw7yNB3lt6R4OFhzlr6PiNMIkIuIAepaZgykQSUOl55bxypJdbD1YyL4jpdT3X1f7YG/KKmvIK63d5yjQ24PCo1V2bZ6/rhe3DoppipJFRFqsM/n9rTlEIg4UE+rDqzf3A6CsspqMvKNUW60YBriZTXQI9cHH4k5JRTVvLN/Luz+mUni0ihBfC3++tBM5xRW8vWoff/lyKwChfha2HijkSEkF/WKCGNwpjOgQnwbXU1Rexf4jpfRqF2h3209EROxphKgBNEIkjSUjr4zktHyu6BGBr6c7hmHwt6+382Fi2kk/0yXcjzfGXkDXCP9TXju/tJLr3lzD/twyEuLCefbaXkQGejm6CyIiLku3zBxMgUiaktVq8Jcvt/HZhgy6hPvRu30goX6erE/NIyWjgGqrQbsgb+ZPHEy4f23AWb8/jw3787l5YDTBvhYqq63c/m4SSal5tuv6e7rzf6PiuHlgtEaLRKRVUCByMAUicQbDME4ILrklFdwwJ5F9R0rp3T6Q/4wfxFsrfuGfq37BMCDYx4PHhndnU3oBn27IwM/TnZdv6M2clftIySgA4LHh3bnv0k4n/c5qq4GHmxagikjzp0DkYApE4kr2Hynl+rfWkldaiZ9n7XwkgMgAL7KKfl22bzbBu3cO5LJu4dRYDd5YvpdXluwG4NWb+nJtv3Z21y0qr+KeDzawIS2fLuF+9GoXyMVd23BV77YaURKRZkmTqkVasI5hvrx9e39u/VcSJRXVBPl48ML1vbk8LpwP1u7n1R/2UFJRzV9H9bDtqu1mNvHA5V0oLq/inR9Tmfq/zYT7ezK4cxhQO9/ojn+vY+vBQgB+zirm56xiPk8+QEZ+GX++tPMJdZRX1bB+fx4H84/i4+mOr8WNuLYBRAV5N91fhoiIg2iEqAE0QiSuaM3eIyzdmcOEi8+zmyydW1JBZmE557cLPOEzVqvB/Z9sYuGWTLw8zPyuSxsGdwrlk3UZ7MouJsTXwuxb+lFWWcOq3Yf5z0+1k7vfHHsBI3u1xWo1+GbLIb5KOUTiL7kcrbJ/TInFzcx/xl/IoPNCG7fzIiINoFtmDqZAJC1JeVUNd3+wgdV7j9gdD/f3ZO7dg+hy3Oq1p77Zzntr9uPpbub/RsbxyfoMdmYW2c5HBHjSo20A5VVWMguPsj+3jDA/TxY+MJSIgFOvaCurrKbaahDg5eGwvuWXVmIyQZCPxWHXFJHmS4HIwRSIpKWxWg22HSpk9d4jrNl7hPIqK3+/oQ8dw3zt2tVYDe75cAPLfs6xHfP3dOeuobGMOD+S7pH+tvlFZZXVXPfGWnZlF9O/QzAf33MRFvcTJ2cfLq7g3dWp/PenNAzDYMaY3lzdJ+q0NS/cksnRqhqu69cON/OJc5oSf8nlrvfX4e/lwaIHf0eYn2c9VxGR1kSByMEUiKQ1K6moZuw7P7Ezs5g74jsw8bLOtseV/FbqkVKunr2a4opqhvWMoEu4P6WV1ZRWVFNaWUNxeTVJ+3Kp+M1z3MYOimHaVT3w8nCr97rfb8/iT/9JBmBQbAh/v7EP7YN/3aByY3o+t/0ribLK2lt411/Qjldu7OuA3otIc6ZA5GAKRNLaVdVYqa4x8LbUH1iOt2RHNvd8uOGUbfpEBzHx0k5sOVDIGyv2YhjQNtCL3u0D6RbhT3ynMOI71c5DOlRwlBGv/Ujh0SpMJjCM2lGqey/tRKc2vriZzUz5LIXi8mp6tQtk26FCDAM+vuci4juFUlBWybMLd+Ln6c6UK7va3aLLL63E08OMj0XrS0RaIgUiB1MgEjkzXyQf4Kd9ufh6uuNjccP32Co0X093YsN86d8h2HarbeXuw0z5NIXcY892q3NV77ZMv6oHkz7axLr9efRuH8grN/Zh6v+2sCm94ITvHNAhmA/HX8izC3fyUVI6ncP9eOXGPkz8aCMZeUcBiAr0YuYf+tA2yIs5K35h/qaD+Hq688DlXbj9og4n3OLLLipnwZZMIgO8GHF+JOZ6btWJiOtSIHIwBSKRxlVSUc2WjAJ+zipm68FCvko5iNWoXbVWWWPF1+LGwgd+R8cwX6prrPz3pzTWp+VzqOAomQXl9G4fyMs39iHAy4PCsip+//cVdgErOsQbN5OJ/bllALaRpuN1DPXhhgHRhPlZ8PP0YPGOLBZuyaTaWtuwT/tA/nZ1Ty6ICW5Qn2qsBgu2HKJ9sA/9OzTsM/Vtxnm88qoa5m08SO/2gfWuIhQRewpEDqZAJNK0th4o5NEvtthWtP3jpj5c1699gz8/b+MBpny2GYDfdQlj9i39sLibeXHRz3xw7DlxV/SI4N5LOrEnu5iXF+/mSElFvdfqEx3ELzkltg0wL+wYQs92AcRFBlB4tIotBwv5ObOIuLYBTB3WjegQH3JLKnjwkxTbSr5LurbhkSu7EeTjwU/7cknJKKBjqC9j+rcnxNdCUXkV//oxlQ8T93NZt3Bm/qH3CbuFF5ZVcc9/NrAuNQ+TCW4eGMOjw7qddD7X6dRYjXonp4u0JApEDqZAJNL0qmqsfLYhAw+zmRsHRp/RZw3D4O1V+3B3M3Pn4I52v/h3HCrC08NMpzZ+tmMlFdX8JzGNPdnF5JdVUnC0ik5t/BgX35Fe7QPJKS7n5e938XnygRNGlo7n6W7mtos68O3WTDILy/HyMFNdY9hGmX7L4mbm0m5tWLc/j4KyKtvxK3tE8PqtF9hu4WUWHmXcv9exO7sET3ezbVJ6oLcHHcN8KSirpKyyhjEXtGfqsG6nDDoV1TW89sMe3l2dyq2DYph+VQ/tRC4tlgKRgykQiQjUrqJLTstnx6EidmUX4efpTu/2QZwX5suHiWkk7su1tT0vzJc5t/fH093Mqz/s4cuUg7iZTPRuH0i/mGCSUnPZdvDXPZ06tfHlun7tmLVsL5XVVhLiIrh1UDRr9+by1eZDHC6uICLAk/fvupDi8mqmf7WNn7OKT6jx993DmXVLP/w8T5wovuNQEVM+S7H73KTLOvPIsG5n/HdhGAYrdx9myY5sbhgQTd/ooDO+hkhjUyByMAUiETkdwzBYtC2L137YQ8+oAJ66pif+x61oKyirxMPNjO9xQWXLgQK+355FpzZ+XN0nCnc3Myt3H+aeDzdQ+ZutCTq18eXD8YNod+zRKNU1Vn7cc4SqGivBvhb2Hynlr19uo6LaSvdIf/44NBYvDzdMwNaDhSTty2XrwUKsBoT4Wriqd1s+PHb7cNpVPRg/NPaEPpVUVPNLTgl7c0rIyC/Dz9OdMD9PKmusvL9mPzuO3dL0cDPxxIg47hrSEajdBmHD/nwGnReqoCROpUDkYApEItKUVu0+zKSPNhLsa2Fwp1DiO4WREBd+2u0BNqXnc8+HySedDwUwrGcEz13XizA/T15ftoeXF9c+8PfC2BDa+HkS4O3Ogfyj7M0pIbOw/KTXAfA59vy65LR8AIZ2DuNQ4VH2HS61tendPpCxg2LoHO5PkI8HHmYzOzIL2ZRRwC85pQR6exAR4EnbIG8u7dqG6JDa/aXKKquZt/Ega/YeISLAi46hPpzXxo/e7QNtO5GvS83jrRV72ZhewP+N7M5NA2Ns35tfWsn2Q0X0iAog5CznWUnzp0DkYApEItJcHMgvY9bSPeQUV1BRZaWqxkqXCD8ujA3hwthQ2wgT1I5qPbtwJ++uTj3p9cL8POkc7kuHEF+OVtVwpKSC0opqft89gjviOxDk48EHa/fz3Lc7qaqp/XXi7eFG/w7BrEvNo7LGetJr16dPdBC92gXwzeZMCo9W1dumc7gfPhY3thwotDs+4eLzeHRYN+ZtPMjzi3ZSUFa7d1W/6CAuj4vghgHtCff/9ZEyGXllbD9USGSgNx1CfAjy8TjpfKqK6hpmfreLH/ccplMbP/pEB9EvOogLOgSfMAH+TJQem6zvW88tzuYmOS2fQG8POof7nb5xE1EgcjAFIhFpqQzDYMuBQtLyysgtqaCgrIqoIC86h/vRqY1fg58LtzmjgA8S9zMoNoRRvaPw83Qnt6SCzzYc4LvtWeSV1l77aGUNXSL86RsdRFxbf4rLq8kpKmdXdjHrUvM4fv55TIgPf+jfntKKalKPlLI7u9i2dQLUTkof0789gd4ezFn5CwBt/D05XFw7Qhbs40H+cZPVLW5mrr+gHUO7hDF/40GW7cqxmyQf6mvhsu7hDO8ZydAuYbad0zMLj3LvfzeyOaPghH77e7lzSdc2DOwYAtQuBnA3m4ht40enNr5UVFv5blsW323L4nBxBRfGhnBx1zb4WNz4ZvMhlv6cQ43VYOJlnZl0Wed6H3fjCFmF5XyZcpD8skq6hvvTLdKfLhF+eLqffrPV0zlSUsHfvt7Owi2ZeHu48fWkIXbPRHQmBSIHUyASEXGMU+21lFNcznfbsthxqIjLuoeTEBdxwoq53JIKNqUXkFVUzpU9Igg/9hDhr1IOMvV/W6istuLt4caUK7py15COHC6pYPnPh/k8OaPeDT3j2gaQV1pBdpH9bUaLm5mOYT7EhvmSnJbPkZJKAr09+L+R3SkoqyIlo4Ck1DzyfrOh6Lno0TaAJ0Z2xzAgv6ySgwVH2Ztdwu6cYkyY+H33cIb1jMTfy51F2zJZtC2Lg/lH8fN0x9/LnYgAL/p3CGZAxxDC/T3JyCsjLa+MJTuyWbErh98udgz1tTDp9525dVBMvcGoqsZKZkE5h0sqyC2pIK5tgO2WZp1vNh9i+lfb7IJn53A/vpo4xDbqte1gIT4WN85rYz9ytDenhBBfS6Pe0lQgcjAFIhER17f1QCHfb8/i5guj7Z51Vyc5LY+3V+3j56xiEuIiGDsoxvZL+mhlDSkZtZPcv9+edcL8qbi2Afzztv7EhP563RqrQUpGAct+zmZvTgnubmYsbmbbiFZabhlWw2Bw5zBGnB9JhxAf1vxyhFW7j1BaUc0VPSO4uk8U+w6XMu2rbXZbLzSGgR2D6R4ZwO7sYn7OKrbdkowO8eaRK7sxuneUbTf2tb8c4eHPNtv9PVjczLx2c19G9GoLwJsr9jLzu122v58nRnTnkc83k1NcwbV9o/jLqB48u3AHX6UcAmpXQN49NJa8skr+vTqVjekFhPl58tmfLjohLDmKApGDKRCJiLQehmGQkXeUfUdKSD1SipvZxA39oxv0LL/jVddYqbYaJ31o8fFyist5ZsFOkvfnEeDtQbCPhYgAT7pE+NM53I/i8mq+357Fqt2HqaqxMig2lJG9IukbHczRqhqKy6v45XAJ6/fns2F/HqUVNbQP8SY62IeeUQGM6d/ebu+t6horn204wKs/7Cbn2C3GHm0DmDq8GynpBcxatgfDqN1bq42/J+7m2p3ezSZ49tpeHMgv480Vtbcp/3TxeTwyrBsebmaS9uVy67+SqLEaeHu4cbSqhroBwZOljahALz6/b7Dd/DZHUSByMAUiERFxBUcra6iyWu0eUvxbdb/WG7Lh5tHKGv69JpU5K36h+NgE7zo3DmjPk1f3xMfiTo3V4K9fbuXjdRl2bR4f0Z17L+lkd+ytFb/w4nc/A9AzKoAZ1/fC38uDd1fv4/MNB/DzdGfsRR0YcX4kEz/ayL7DpXQM9eGze+PtJr07ggKRgykQiYhIS5ZfWskby/fyYWIa7m4mnrvu/BMel2MYBn9fvJvXl+/FZIKnrzmf2y/qcMK1rFaDf69Jxcfizo0D2uN+3Cq86horJpPJNjfsUMFRbpiTyMGCo3SL8OfTP13U4In8DaFA5GAKRCIi0hrkllRgMplOOdF5+a4cfC3uXBgb4pDvTMst5YY5iXSL9Oft2wec8a3JU1EgcjAFIhERkcaTnltGRKCnQ7YBON6Z/P5u/jtBiYiISLN2/Oo9Z2mcHaBEREREmhEFIhEREWn1FIhERESk1VMgEhERkVZPgUhERERavVYViN544w06duyIl5cXgwYNYt26dc4uSURERFxAqwlEn376KVOmTOFvf/sbGzdupE+fPgwbNoycnBxnlyYiIiJO1mo2Zhw0aBADBw7k9ddfB8BqtRIdHc3999/P448/bte2oqKCiooK2/uioiKio6O1MaOIiEgzciYbM7aKEaLKykqSk5NJSEiwHTObzSQkJJCYmHhC+xkzZhAYGGh7RUdHN2W5IiIi0sRaRSA6cuQINTU1RERE2B2PiIggKyvrhPZPPPEEhYWFtldGRsYJbURERKTl0KM76uHp6Ymnp6ezyxAREZEm0ipGiMLCwnBzcyM7O9vueHZ2NpGRkU6qSkRERFxFqwhEFouF/v37s3TpUtsxq9XK0qVLiY+Pd2JlIiIi4gpazS2zKVOmMG7cOAYMGMCFF17Iq6++SmlpKXfddddpP1u3EK+oqKixyxQREREHqfu93ZAF9a0mEN10000cPnyY6dOnk5WVRd++ffnuu+9OmGhdn+LiYgCtNhMREWmGiouLCQwMPGWbVrMP0bmwWq0cOnQIf39/TCaTs8txuLp9ljIyMlrFPkutqb+tqa+g/rZkramvoP46imEYFBcXExUVhdl86llCrWaE6FyYzWbat2/v7DIaXUBAQKv4D69Oa+pva+orqL8tWWvqK6i/jnC6kaE6rWJStYiIiMipKBCJiIhIq6dAJHh6evK3v/2t1WxG2Zr625r6CupvS9aa+grqrzNoUrWIiIi0ehohEhERkVZPgUhERERaPQUiERERafUUiERERKTVUyBqJWbMmMHAgQPx9/cnPDyca6+9ll27dtm1KS8vZ+LEiYSGhuLn58eYMWPIzs52UsWO88ILL2AymZg8ebLtWEvr68GDB7ntttsIDQ3F29ubXr16sWHDBtt5wzCYPn06bdu2xdvbm4SEBPbs2ePEis9eTU0N06ZNIzY2Fm9vbzp16sQzzzxj96yi5tzfVatWMXr0aKKiojCZTHz55Zd25xvSt7y8PMaOHUtAQABBQUGMHz+ekpKSJuxFw52qv1VVVTz22GP06tULX19foqKiuOOOOzh06JDdNVpKf3/r3nvvxWQy8eqrr9odby79bUhfd+7cydVXX01gYCC+vr4MHDiQ9PR02/mm/FmtQNRKrFy5kokTJ/LTTz+xZMkSqqqquPLKKyktLbW1eeihh/jmm2/4/PPPWblyJYcOHeL66693YtXnbv369fzzn/+kd+/edsdbUl/z8/MZMmQIHh4eLFq0iB07dvD3v/+d4OBgW5uZM2cya9Ys5syZQ1JSEr6+vgwbNozy8nInVn52XnzxRd566y1ef/11du7cyYsvvsjMmTOZPXu2rU1z7m9paSl9+vThjTfeqPd8Q/o2duxYtm/fzpIlS1iwYAGrVq1iwoQJTdWFM3Kq/paVlbFx40amTZvGxo0bmTdvHrt27eLqq6+2a9dS+nu8+fPn89NPPxEVFXXCuebS39P19ZdffmHo0KF0796dFStWsGXLFqZNm4aXl5etTZP+rDakVcrJyTEAY+XKlYZhGEZBQYHh4eFhfP7557Y2O3fuNAAjMTHRWWWek+LiYqNLly7GkiVLjEsuucR48MEHDcNoeX197LHHjKFDh570vNVqNSIjI42XXnrJdqygoMDw9PQ0Pv7446Yo0aFGjRpl/PGPf7Q7dv311xtjx441DKNl9Rcw5s+fb3vfkL7t2LHDAIz169fb2ixatMgwmUzGwYMHm6z2s/Hb/tZn3bp1BmCkpaUZhtEy+3vgwAGjXbt2xrZt24wOHToY//jHP2znmmt/6+vrTTfdZNx2220n/UxT/6zWCFErVVhYCEBISAgAycnJVFVVkZCQYGvTvXt3YmJiSExMdEqN52rixImMGjXKrk/Q8vr69ddfM2DAAG644QbCw8Pp168f77zzju18amoqWVlZdv0NDAxk0KBBzbK/gwcPZunSpezevRuAzZs3s3r1akaMGAG0vP4eryF9S0xMJCgoiAEDBtjaJCQkYDabSUpKavKaHa2wsBCTyURQUBDQ8vprtVq5/fbbmTp1Kj179jzhfEvpr9VqZeHChXTt2pVhw4YRHh7OoEGD7G6rNfXPagWiVshqtTJ58mSGDBnC+eefD0BWVhYWi8X2Q6ZOREQEWVlZTqjy3HzyySds3LiRGTNmnHCupfV13759vPXWW3Tp0oXvv/+e++67jwceeIAPPvgAwNaniIgIu8811/4+/vjj3HzzzXTv3h0PDw/69evH5MmTGTt2LNDy+nu8hvQtKyuL8PBwu/Pu7u6EhIQ0+/6Xl5fz2GOPccstt9geANrS+vviiy/i7u7OAw88UO/5ltLfnJwcSkpKeOGFFxg+fDiLFy/muuuu4/rrr2flypVA0/+s1tPuW6GJEyeybds2Vq9e7exSGkVGRgYPPvggS5YssbsX3VJZrVYGDBjA888/D0C/fv3Ytm0bc+bMYdy4cU6uzvE+++wz5s6dy0cffUTPnj1JSUlh8uTJREVFtcj+Sq2qqipuvPFGDMPgrbfecnY5jSI5OZnXXnuNjRs3YjKZnF1Oo7JarQBcc801PPTQQwD07duXtWvXMmfOHC655JImr0kjRK3MpEmTWLBgAcuXL6d9+/a245GRkVRWVlJQUGDXPjs7m8jIyCau8twkJyeTk5PDBRdcgLu7O+7u7qxcuZJZs2bh7u5OREREi+krQNu2benRo4fdsbi4ONtKjbo+/XZlRnPt79SpU22jRL169eL222/noYceso0GtrT+Hq8hfYuMjCQnJ8fufHV1NXl5ec22/3VhKC0tjSVLlthGh6Bl9ffHH38kJyeHmJgY28+utLQ0Hn74YTp27Ai0nP6GhYXh7u5+2p9dTfmzWoGolTAMg0mTJjF//nyWLVtGbGys3fn+/fvj4eHB0qVLbcd27dpFeno68fHxTV3uObn88svZunUrKSkptteAAQMYO3as7c8tpa8AQ4YMOWELhd27d9OhQwcAYmNjiYyMtOtvUVERSUlJzbK/ZWVlmM32P7rc3Nxs/8fZ0vp7vIb0LT4+noKCApKTk21tli1bhtVqZdCgQU1e87mqC0N79uzhhx9+IDQ01O58S+rv7bffzpYtW+x+dkVFRTF16lS+//57oOX012KxMHDgwFP+7Gry30sOn6YtLum+++4zAgMDjRUrVhiZmZm2V1lZma3Nvffea8TExBjLli0zNmzYYMTHxxvx8fFOrNpxjl9lZhgtq6/r1q0z3N3djeeee87Ys2ePMXfuXMPHx8f473//a2vzwgsvGEFBQcZXX31lbNmyxbjmmmuM2NhY4+jRo06s/OyMGzfOaNeunbFgwQIjNTXVmDdvnhEWFmY8+uijtjbNub/FxcXGpk2bjE2bNhmA8corrxibNm2yrapqSN+GDx9u9OvXz0hKSjJWr15tdOnSxbjllluc1aVTOlV/Kysrjauvvtpo3769kZKSYvezq6KiwnaNltLf+vx2lZlhNJ/+nq6v8+bNMzw8PIy3337b2LNnjzF79mzDzc3N+PHHH23XaMqf1QpErQRQ7+u9996ztTl69Kjx5z//2QgODjZ8fHyM6667zsjMzHRe0Q7020DU0vr6zTffGOeff77h6elpdO/e3Xj77bftzlutVmPatGlGRESE4enpaVx++eXGrl27nFTtuSkqKjIefPBBIyYmxvDy8jLOO+884y9/+YvdL8jm3N/ly5fX+9/quHHjDMNoWN9yc3ONW265xfDz8zMCAgKMu+66yyguLnZCb07vVP1NTU096c+u5cuX267RUvpbn/oCUXPpb0P6+u677xqdO3c2vLy8jD59+hhffvml3TWa8me1yTCO295VREREpBXSHCIRERFp9RSIREREpNVTIBIREZFWT4FIREREWj0FIhEREWn1FIhERESk1VMgEhERkVZPgUhERERaPQUiEXFZHTt25NVXX21w+xUrVmAymU54GKSIyOlop2oRcZhLL72Uvn37nlGIOZXDhw/j6+uLj49Pg9pXVlaSl5dHREQEJpPJITWcqRUrVnDZZZeRn59PUFCQU2oQkTPn7uwCRKR1MQyDmpoa3N1P/+OnTZs2Z3Rti8VCZGTk2ZYmIq2YbpmJiEPceeedrFy5ktdeew2TyYTJZGL//v2221iLFi2if//+eHp6snr1an755ReuueYaIiIi8PPzY+DAgfzwww921/ztLTOTycS//vUvrrvuOnx8fOjSpQtff/217fxvb5m9//77BAUF8f333xMXF4efnx/Dhw8nMzPT9pnq6moeeOABgoKCCA0N5bHHHmPcuHFce+21J+1rWloao0ePJjg4GF9fX3r27Mm3337L/v37ueyyywAIDg7GZDJx5513AmC1WpkxYwaxsbF4e3vTp08f/ve//51Q+8KFC+nduzdeXl5cdNFFbNu27bTfKyLnToFIRBzitddeIz4+nnvuuYfMzEwyMzOJjo62nX/88cd54YUX2LlzJ71796akpISRI0eydOlSNm3axPDhwxk9ejTp6emn/J6nnnqKG2+8kS1btjBy5EjGjh1LXl7eSduXlZXx8ssv85///IdVq1aRnp7OI488Yjv/4osvMnfuXN577z3WrFlDUVERX3755SlrmDhxIhUVFaxatYqtW7fy4osv4ufnR3R0NF988QUAu3btIjMzk9deew2AGTNm8OGHHzJnzhy2b9/OQw89xG233cbKlSvtrj116lT+/ve/s379etq0acPo0aOpqqo65feKiAMYIiIOcskllxgPPvig3bHly5cbgPHll1+e9vM9e/Y0Zs+ebXvfoUMH4x//+IftPWD89a9/tb0vKSkxAGPRokV235Wfn28YhmG89957BmDs3bvX9pk33njDiIiIsL2PiIgwXnrpJdv76upqIyYmxrjmmmtOWmevXr2MJ598st5zv63BMAyjvLzc8PHxMdauXWvXdvz48cYtt9xi97lPPvnEdj43N9fw9vY2Pv3009N+r4icG80hEpEmMWDAALv3JSUlPPnkkyxcuJDMzEyqq6s5evToaUeIevfubfuzr68vAQEB5OTknLS9j48PnTp1sr1v27atrX1hYSHZ2dlceOGFtvNubm70798fq9V60ms+8MAD3HfffSxevJiEhATGjBljV9dv7d27l7KyMq644gq745WVlfTr18/uWHx8vO3PISEhdOvWjZ07d57V94pIw+mWmYg0CV9fX7v3jzzyCPPnz+f555/nxx9/JCUlhV69elFZWXnK63h4eNi9N5lMpwwv9bU3znFx7d13382+ffu4/fbb2bp1KwMGDGD27NknbV9SUgLAwoULSUlJsb127NhhN4/I0d8rIg2nQCQiDmOxWKipqWlQ2zVr1nDnnXdy3XXX0atXLyIjI9m/f3/jFvgbgYGBREREsH79etuxmpoaNm7ceNrPRkdHc++99zJv3jwefvhh3nnnHaD276DuOnV69OiBp6cn6enpdO7c2e51/DwrgJ9++sn25/z8fHbv3k1cXNxpv1dEzo1umYmIw3Ts2JGkpCT279+Pn58fISEhJ23bpUsX5s2bx+jRozGZTEybNu2UIz2N5f7772fGjBl07tyZ7t27M3v2bPLz80+5j9HkyZMZMWIEXbt2JT8/n+XLl9tCS4cOHTCZTCxYsICRI0fi7e2Nv78/jzzyCA899BBWq5WhQ4dSWFjImjVrCAgIYNy4cbZrP/3004SGhhIREcFf/vIXwsLCbCveTvW9InJuNEIkIg7zyCOP4ObmRo8ePWjTps0p5wO98sorBAcHM3jwYEaPHs2wYcO44IILmrDaWo899hi33HILd9xxB/Hx8fj5+TFs2DC8vLxO+pmamhomTpxIXFwcw4cPp2vXrrz55psAtGvXjqeeeorHH3+ciIgIJk2aBMAzzzzDtGnTmDFjhu1zCxcuJDY21u7aL7zwAg8++CD9+/cnKyuLb775xm7U6WTfKyLnRjtVi4gcx2q1EhcXx4033sgzzzzTZN+rHa5FnEu3zESkVUtLS2Px4sVccsklVFRU8Prrr5Oamsqtt97q7NJEpAnplpmItGpms5n333+fgQMHMmTIELZu3coPP/yguTkirYxumYmIiEirpxEiERERafUUiERERKTVUyASERGRVk+BSERERFo9BSIRERFp9RSIREREpNVTIBIREZFWT4FIREREWr3/B8Otn7lYrTwmAAAAAElFTkSuQmCC",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import numpy as np\n",
    "from torch.utils.data import Dataset, DataLoader\n",
    "from torch.optim import SGD, Adam\n",
    "from copy import deepcopy\n",
    "\n",
    "# 将数据适配为PyTorch数据集\n",
    "class SeqLabelDataset(Dataset):\n",
    "    def __init__(self, tokens, labels):\n",
    "        self.tokens = deepcopy(tokens)\n",
    "        self.labels = deepcopy(labels)\n",
    "        \n",
    "    def __getitem__(self, idx):\n",
    "        return self.tokens[idx], self.labels[idx]\n",
    "    \n",
    "    def __len__(self):\n",
    "        return len(self.labels)\n",
    "    \n",
    "    # 将每个批次转化为PyTorch张量\n",
    "    @classmethod\n",
    "    def collate_batch(cls, inputs):\n",
    "        input_ids, labels, masks = [], [], []\n",
    "        max_len = -1\n",
    "        for ids, tags in inputs:\n",
    "            input_ids.append(ids)\n",
    "            labels.append(tags)\n",
    "            masks.append([1] * len(tags))\n",
    "            max_len = max(max_len, len(ids))\n",
    "        for ids, tags, msks in zip(input_ids, labels, masks):\n",
    "            pad_len = max_len - len(ids)\n",
    "            ids.extend([0] * pad_len)\n",
    "            tags.extend([0] * pad_len)\n",
    "            msks.extend([0] * pad_len)\n",
    "        input_ids = torch.tensor(np.array(input_ids),\\\n",
    "            dtype=torch.long)\n",
    "        labels = torch.tensor(np.array(labels), dtype=torch.long)\n",
    "        masks = torch.tensor(np.array(masks), dtype=torch.uint8)\n",
    "        return {'input_ids': input_ids, 'masks': masks,\\\n",
    "                'labels': labels}\n",
    "    \n",
    "# 准备数据\n",
    "train_dataset = SeqLabelDataset(train_X, train_Y)\n",
    "test_dataset = SeqLabelDataset(test_X, test_Y)\n",
    "\n",
    "from tqdm import tqdm, trange\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "def train(model, batch_size, epochs, learning_rate):\n",
    "    train_dataloader = DataLoader(train_dataset,\n",
    "        batch_size=batch_size, shuffle=True, \n",
    "        collate_fn=SeqLabelDataset.collate_batch)\n",
    "    optimizer = Adam(model.parameters(), lr=learning_rate)\n",
    "    model.zero_grad()\n",
    "    tr_step = []\n",
    "    tr_loss = []\n",
    "    global_step = 0\n",
    "    \n",
    "    with trange(epochs, desc='epoch', ncols=60) as pbar:\n",
    "        for epoch in pbar:\n",
    "            model.train()\n",
    "            for step, batch in enumerate(train_dataloader):\n",
    "                global_step += 1\n",
    "                loss = model(**batch)\n",
    "                loss.backward()\n",
    "                optimizer.step()\n",
    "                model.zero_grad()\n",
    "                pbar.set_description(f'epoch-{epoch}, '+\\\n",
    "                    f'loss={loss.item():.2f}')\n",
    "                \n",
    "                if epoch > 0:\n",
    "                    tr_step.append(global_step)\n",
    "                    tr_loss.append(loss.item())\n",
    "\n",
    "    # 打印损失曲线\n",
    "    plt.plot(tr_step, tr_loss, label='train loss')\n",
    "    plt.xlabel('training steps')\n",
    "    plt.ylabel('loss')\n",
    "    plt.legend()\n",
    "    plt.show()\n",
    "                \n",
    "vocab_size = len(dataset.token2id)\n",
    "hidden_size = 32\n",
    "n_tags = len(dataset.label2id)\n",
    "crf = CRF(vocab_size, hidden_size, n_tags)\n",
    "batch_size = 128\n",
    "epochs = 20\n",
    "learning_rate = 1e-2\n",
    "train(crf, batch_size, epochs, learning_rate)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "4a1e7e25",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "precision = 0.5387509405568096, recall = 0.25608011444921314, f1 = 0.3471515151515151\n",
      "precision = 0.4539800995024876, recall = 0.22351500306184935, f1 = 0.29954862535904797\n"
     ]
    }
   ],
   "source": [
    "# 验证效果\n",
    "\n",
    "def evaluate(X, Y, model, batch_size):\n",
    "    dataset = SeqLabelDataset(X, Y)\n",
    "    dataloader = DataLoader(dataset, batch_size=batch_size,\\\n",
    "        collate_fn=SeqLabelDataset.collate_batch)\n",
    "    model.eval()\n",
    "    with torch.no_grad():\n",
    "        P = []\n",
    "        for batch in dataloader:\n",
    "            preds = model.decode(batch['input_ids'], batch['masks'])\n",
    "            P.extend(preds)\n",
    "\n",
    "    p, r, f = compute_metric(Y, P)\n",
    "    print(f'precision = {p}, recall = {r}, f1 = {f}')\n",
    "    \n",
    "evaluate(train_X, train_Y, crf, batch_size)\n",
    "evaluate(test_X, test_Y, crf, batch_size)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "cc369a17",
   "metadata": {},
   "source": [
    "\n",
    "\n",
    "这里介绍基于双向长短期记忆-条件随机场（BiLSTM-CRF）结构的代码实现。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "990c5bc1",
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "from torch import nn\n",
    "\n",
    "# 定义模型，将长短期记忆的输出输入给条件随机场，\n",
    "# 利用条件随机场计算损失和解码\n",
    "class LSTM_CRF(nn.Module):\n",
    "    def __init__(self, vocab_size, hidden_size, num_layers,\\\n",
    "                 dropout, n_tags):\n",
    "        super().__init__()\n",
    "        self.embedding = nn.Embedding(vocab_size, hidden_size)\n",
    "        self.lstm = nn.LSTM(input_size=hidden_size,\n",
    "                            hidden_size=hidden_size,\n",
    "                            num_layers=num_layers,\n",
    "                            batch_first=False,\n",
    "                            dropout=dropout,\n",
    "                            bidirectional=True)\n",
    "        self.crf = CRFLayer(n_tags, hidden_size * 2)\n",
    "    \n",
    "    def forward(self, input_ids, masks, labels):\n",
    "        \"\"\"\n",
    "        input_ids/masks/labels: batch_size * seq_len\n",
    "        \"\"\"\n",
    "        # 将输入序列转化为词嵌入序列\n",
    "        # batch_size * seq_len * embed_size\n",
    "        embed = self.embedding(input_ids)\n",
    "        embed = torch.transpose(embed, 0, 1)\n",
    "        masks = torch.transpose(masks, 0, 1)\n",
    "        labels = torch.transpose(labels, 0, 1)\n",
    "        # 输入长短期记忆得到隐状态\n",
    "        hidden_states, _ = self.lstm(embed)\n",
    "        # 隐状态和标签、掩码输入条件随机场计算损失\n",
    "        llh = self.crf(hidden_states, labels, masks)\n",
    "        return -llh\n",
    "    \n",
    "    def decode(self, input_ids, masks):\n",
    "        # 输入长短期记忆得到隐状态\n",
    "        embed = self.embedding(input_ids)\n",
    "        embed = torch.transpose(embed, 0, 1)\n",
    "        masks = torch.transpose(masks, 0, 1)\n",
    "        hidden_states, _ = self.lstm(embed)\n",
    "        # 调用条件随机场进行解码\n",
    "        return self.crf.decode(hidden_states, masks)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "ab41a9f0",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "epoch-19, loss=35.48: 100%|█| 20/20 [01:57<00:00,  5.90s/it]\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAjsAAAGwCAYAAABPSaTdAAAAOnRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjEwLjEsIGh0dHBzOi8vbWF0cGxvdGxpYi5vcmcvc2/+5QAAAAlwSFlzAAAPYQAAD2EBqD+naQAAcWVJREFUeJzt3Xd4lFX+/vH3lGTSK6RB6B1CR4yooKIgio1dV8W2a/npwqpY1vW7q2tZRd1dO+JWy66uFSxY6YgUkSJIL4FQUiAhvc88vz9m5kkGQhIgdXK/rivXZmaezJxH3HD7OZ9zjsUwDAMRERERP2Vt6QGIiIiINCWFHREREfFrCjsiIiLi1xR2RERExK8p7IiIiIhfU9gRERERv6awIyIiIn7N3tIDaA1cLheHDh0iPDwci8XS0sMRERGRBjAMg8LCQpKSkrBaT1y/UdgBDh06RHJycksPQ0RERE7B/v376dy58wlfV9gBwsPDAfc/rIiIiBYejYiIiDREQUEBycnJ5t/jJ6KwA+bUVUREhMKOiIhIG1NfC4oalEVERMSvKeyIiIiIX1PYEREREb+mnh0REfF7TqeTysrKlh6GnKSAgABsNttpv4/CjoiI+C3DMMjMzCQvL6+lhyKnKCoqioSEhNPaB09hR0RE/JY36MTFxRESEqKNY9sQwzAoKSkhOzsbgMTExFN+L4UdERHxS06n0ww6sbGxLT0cOQXBwcEAZGdnExcXd8pTWmpQFhERv+Tt0QkJCWnhkcjp8P75nU7PlcKOiIj4NU1dtW2N8eensCMiIiJ+rUXDzuzZsxk8eLB5TENqaipffvml+XpZWRnTpk0jNjaWsLAwpkyZQlZWls97pKenc8kllxASEkJcXBwPPPAAVVVVzX0rIiIi0kq1aNjp3LkzTz/9NGvXruWHH37g/PPP5/LLL2fz5s0AzJgxg88++4wPPviApUuXcujQIa666irz551OJ5dccgkVFRWsWLGCN998kzfeeINHHnmkpW5JRESk1enWrRsvvPBCi79HizFamejoaOOf//ynkZeXZwQEBBgffPCB+drWrVsNwFi5cqVhGIbxxRdfGFar1cjMzDSvmT17thEREWGUl5c3+DPz8/MNwMjPz2+8G6mDy+UySiuqmuWzRETaq9LSUmPLli1GaWlpSw/lpI0dO9a4++67G+39srOzjeLi4tN6j65duxrPP/984wzoJNT159jQv79bTc+O0+nk3Xffpbi4mNTUVNauXUtlZSXjx483r+nXrx9dunRh5cqVAKxcuZKUlBTi4+PNayZMmEBBQYFZHapNeXk5BQUFPl/N6ddvr+OMJxeQU1TerJ8rIiL+wzCMBrdtdOzYsV2vSmvxsLNp0ybCwsJwOBzccccdzJ07lwEDBpCZmUlgYCBRUVE+18fHx5OZmQm4N4uqGXS8r3tfO5GZM2cSGRlpfiUnJzfuTdVjzd5cCsqq2J5V2KyfKyLS3hmGQUlFVYt8GYbRoDHefPPNLF26lBdffBGLxYLFYmHv3r0sWbIEi8XCl19+yYgRI3A4HCxfvpzdu3dz+eWXEx8fT1hYGKNGjWLBggU+73nsFJTFYuGf//wnV155JSEhIfTu3ZtPP/30pP5Zpqenc/nllxMWFkZERARXX321T1/tjz/+yHnnnUd4eDgRERGMGDGCH374AYB9+/YxefJkoqOjCQ0NZeDAgXzxxRcn9fkno8U3Fezbty8bNmwgPz+fDz/8kJtuuomlS5c26Wc+9NBD3HvvvebjgoKCZgs8hmFQUOpO4vklOqdFRKQ5lVY6GfDI1y3y2Vsen0BIYP1/7b744ovs2LGDQYMG8fjjjwPuyszevXsB+N3vfsdf/vIXevToQXR0NPv372fSpEk8+eSTOBwO3nrrLSZPnsz27dvp0qXLCT/nscce49lnn+XPf/4zL7/8MlOnTmXfvn3ExMTUO0aXy2UGnaVLl1JVVcW0adP4xS9+wZIlSwCYOnUqw4YNY/bs2dhsNjZs2EBAQAAA06ZNo6KigmXLlhEaGsqWLVsICwur93NPVYuHncDAQHr16gXAiBEjWLNmDS+++CK/+MUvqKioIC8vz6e6k5WVRUJCAgAJCQl8//33Pu/nTZXea2rjcDhwOByNfCcNU17losLpAiC/VGFHRER8RUZGEhgYSEhISK1/lz3++ONceOGF5uOYmBiGDBliPn7iiSeYO3cun376KdOnTz/h59x8881ce+21ADz11FO89NJLfP/990ycOLHeMS5cuJBNmzaRlpZmFgveeustBg4cyJo1axg1ahTp6ek88MAD9OvXD4DevXubP5+ens6UKVNISUkBoEePHvV+5ulo8bBzLJfLRXl5OSNGjCAgIICFCxcyZcoUALZv3056ejqpqakApKam8uSTT5rbSAPMnz+fiIgIBgwY0GL3UJeCGgEnT2FHRKRZBQfY2PL4hBb77MYwcuRIn8dFRUU8+uijfP7552RkZFBVVUVpaSnp6el1vs/gwYPN70NDQ4mIiDDPoarP1q1bSU5O9pkVGTBgAFFRUWzdupVRo0Zx7733cuutt/Kf//yH8ePH8/Of/5yePXsCcNddd3HnnXfyzTffMH78eKZMmeIznsbWoj07Dz30EMuWLWPv3r1s2rSJhx56iCVLljB16lQiIyO55ZZbuPfee1m8eDFr167ll7/8JampqZx55pkAXHTRRQwYMIAbbriBH3/8ka+//po//OEPTJs2rcUqN/UpKKsOOKrsiIg0L4vFQkigvUW+Gmsn59DQUJ/H999/P3PnzuWpp57i22+/ZcOGDaSkpFBRUVHn+3inlGr+s3G5XI0yRoBHH32UzZs3c8kll7Bo0SIGDBjA3LlzAbj11lvZs2cPN9xwA5s2bWLkyJG8/PLLjfbZx2rRsJOdnc2NN95I3759ueCCC1izZg1ff/21WZ57/vnnufTSS5kyZQrnnnsuCQkJzJkzx/x5m83GvHnzsNlspKamcv3113PjjTeac5ytUX5pded8nnp2RESkFoGBgTidzgZd+91333HzzTdz5ZVXkpKSQkJCgtnf01T69+/P/v372b9/v/ncli1byMvL85lZ6dOnDzNmzOCbb77hqquu4vXXXzdfS05O5o477mDOnDncd999/OMf/2iy8bboNNa//vWvOl8PCgpi1qxZzJo164TXdO3atUk7uBubb2Wn7tQtIiLtU7du3Vi9ejV79+4lLCyszqbh3r17M2fOHCZPnozFYuHhhx9u1ApNbcaPH09KSgpTp07lhRdeoKqqil//+teMHTuWkSNHUlpaygMPPMDPfvYzunfvzoEDB1izZo3ZlnLPPfdw8cUX06dPH44ePcrixYvp379/k423xZeetzc1e3Y0jSUiIrW5//77sdlsDBgwgI4dO9bZf/Pcc88RHR3NWWedxeTJk5kwYQLDhw9v0vFZLBY++eQToqOjOffccxk/fjw9evTgvffeA9wzLzk5Odx444306dOHq6++mosvvpjHHnsMcO+tN23aNPr378/EiRPp06cPr776atON12jown8/VlBQQGRkJPn5+URERDTpZ/1n1T4e/vgnAAYmRfD5Xec06eeJiLRXZWVlpKWl0b17d4KCglp6OHKK6vpzbOjf36rsNDNVdkRERJqXwk4z8+nZUYOyiIhIk1PYaWYFNVZjFZZXUeVs2iYyERGR9k5hp5nVrOy4HzfsEDcRETk1ak1t2xrjz09hp5kVHNOno74dEZGm4d00r6SkpIVHIqfD++d37CaIJ6PVHRfh746t5OSVVAChtV8sIiKnzGazERUVZR6BEBIS0mi7GEvTMwyDkpISsrOziYqKwmY79eM2FHaaWeExlRydjyUi0nS8B2k29MwnaX2ioqLqPNy7IRR2mpm3Zyc2NJCc4orjprVERKTxWCwWEhMTiYuLo7JSv2/bmoCAgNOq6Hgp7DQjwzDM1VjJMSHkFFfofCwRkWZgs9ka5S9NaZvUoNyMyqtcVHiWmifHhABqUBYREWlqCjvNyDtlZbVAUpR7y2tVdkRERJqWwk4z8vbrRAQHEBUcCKiyIyIi0tQUdppRvqdfJyIogKiQAM9zFebrT3+5jav/tpLyKmeLjE9ERMQfKew0o+rKjp2oYHfY8U5juVwGb6xI4/u0XLZnFrbYGEVERPyNwk4z8vbsRAQFEBnsrey4n8suLKes0t28XFyuyo6IiEhjUdhpRt7dkyOCAoj0TGN5NxXcm1NsXldSofOyREREGovCTjMyKzvBdp/KjmEY7KsRdoorVNkRERFpLAo7zcjs2QkKICrEvRqrospFWaWLvTnVB9WVlKuyIyIi0lgUdpqRd/fkiOAAQgNt2KzuA+nySyt9KjtFCjsiIiKNRmGnGVU3KNuxWCzVK7JKK9hXs7KjaSwREZFGo7DTjLzTWN7mZLNJuaTSJ+wUq0FZRESk0SjsNKOaS88Bs0l5z+Fin6mrEi09FxERaTQKO83IXHruCTneaawf9+f5XKfKjoiISONR2GlGJ6rs/Hggz+e6YjUoi4iINBqFnWZiGIbPcRGAufx8R5b7eIiQQBugBmUREZHGpLDTTMoqXVQ6DaC6suOdznK5n6Z/YgSgyo6IiEhjUthpJt6qjs1qMSs43p4dr4FJ7rBzbGWnpKKKw4XlzTBKERER/6Ow00yO3WMHqnt2vAZ4KzvHNChf8/dVnPvsYnKLK5phpCIiIv5FYaeZVPfrVAecqBDfsNPPE3ZqLj03DINtGYWUVjrZmlHQDCMVERHxLwo7zcQ8KiKo9rATF+6gQ5i7YbnmnjvlVS4qnC4An40HRUREpGEUdprJsSuxwHcaq1tsKKGB7tfKq1xUeQKOd/oL8Dk/S0RERBpGYaeZHLvHDkBkcKD5fdfYEEIcNvNxSaV7KssbkkCVHRERkVOhsNNMzN2TfcJOjcpOh1AcdhsBNnfzsrdvJ7+0ekprryo7IiIiJ01hp5mYlZ0a01iBdqu5DL1rbAgAIZ6pLO+KrJqVnfTcEgzDaJbxioiI+AuFnWZi9uwE+a7A6hQVDEC/hHAAQj3hx7uxYM2enZIKJ0eKtPxcRETkZNjrv0Qag7ka65i9dWZNHc6+nBJ6xbnDTojDU9kp9/bs+O65sy+nmI7hjqYeroiIiN9QZaeZ1LYaC6BPfDgXDog3H4ea52MdX9kBNSmLiIicLIWdZlLbaqzahHorOxXHr8YC2JersCMiInIyFHaaSV7p8Tso18bboFxi9uy4/9db8dFeOyIiIidHYacZuFwGGfllACREBNV5bahnr51jKzsDO0UCmsYSERE5WQo7zeBIUTkVVS6sFkiIrDvsmEvPPZWdQk+DcooZdlTZERERORkKO83gQF4pAImRwQTY6v5Hbi49P6ZB2Rt2jpZUHtfHIyIiIiemsNMMDhx1hx3vnjp18S49Lyn3ncZKjAwyDwpN11SWiIhIgynsNIODnrDTObr+sBPmOLayU70/T5cY9y7LOjZCRESk4RR2msGBo+5KTKcGhJ3q1Vi+lZ2I4AC6xoYCalIWERE5GQo7zeDASVR2QmtUdsoqnVRUuQAID7Kb52dpGktERKThFHaawcE8b89OSL3X1lyN5a3qWCwQFlgddjSNJSIi0nAKO03MMAxzGqtBlR3vNFaF0+zXCXfYsVot5jRWunZRFhERaTCFnSaWW1xBWaV7Kioxqu49dsB3Gqtmvw5AV0+DckZ+GWWVzqYYroiIiN9R2Gli3n6d+AgHDrut3utDayw9P/Y8rZjQQEI8+/Ac8kyNiYiISN1aNOzMnDmTUaNGER4eTlxcHFdccQXbt2/3uWbcuHFYLBafrzvuuMPnmvT0dC655BJCQkKIi4vjgQceoKqqqjlv5YSq+3Xqn8ICzDDjrux4l527A5DFYiHSU+UpKm8d9yciItLa2Vvyw5cuXcq0adMYNWoUVVVV/N///R8XXXQRW7ZsITQ01Lzutttu4/HHHzcfh4RUN/o6nU4uueQSEhISWLFiBRkZGdx4440EBATw1FNPNev91Ka6X6f+5mSo7tkpq3SRV1IB+J6UHuap/BSVKeyIiIg0RIuGna+++srn8RtvvEFcXBxr167l3HPPNZ8PCQkhISGh1vf45ptv2LJlCwsWLCA+Pp6hQ4fyxBNP8OCDD/Loo48SGBh43M+Ul5dTXl5uPi4oKGikOzqeuXtyA5qTAUIc1VNdmZ7DQ2uelB4W5P4jK1RlR0REpEFaVc9Ofn4+ADExMT7Pv/3223To0IFBgwbx0EMPUVJSvRpp5cqVpKSkEB8fbz43YcIECgoK2Lx5c62fM3PmTCIjI82v5OTkJrgbt5PZPRnAYbcRYLMANcJOjcpOuOd7VXZEREQapkUrOzW5XC7uuecexowZw6BBg8znr7vuOrp27UpSUhIbN27kwQcfZPv27cyZMweAzMxMn6ADmI8zMzNr/ayHHnqIe++913xcUFDQZIGnekPBhk1jgXuvnfzSSjLMyk71H1O4dxpLlR0REZEGaTVhZ9q0afz0008sX77c5/nbb7/d/D4lJYXExEQuuOACdu/eTc+ePU/psxwOBw6H47TG2xCGYZx0gzK4Tz7PL60ks+D4yo63Z6dQJ5+LiIg0SKuYxpo+fTrz5s1j8eLFdO7cuc5rR48eDcCuXbsASEhIICsry+ca7+MT9fk0l/zSSrMC09BpLKg++Twj3x2U1LMjIiJy6lo07BiGwfTp05k7dy6LFi2ie/fu9f7Mhg0bAEhMTAQgNTWVTZs2kZ2dbV4zf/58IiIiGDBgQJOMu6G8U1gdwgIJCqh/jx2vUM/yc+9mhOFB1QU4rcYSERE5OS06jTVt2jTeeecdPvnkE8LDw80em8jISIKDg9m9ezfvvPMOkyZNIjY2lo0bNzJjxgzOPfdcBg8eDMBFF13EgAEDuOGGG3j22WfJzMzkD3/4A9OmTWuWqaq6VK/Eani/DlSfj+Xl26Csnh0REZGT0aKVndmzZ5Ofn8+4ceNITEw0v9577z0AAgMDWbBgARdddBH9+vXjvvvuY8qUKXz22Wfme9hsNubNm4fNZiM1NZXrr7+eG2+80WdfnpZi7rFzEv06UL2LslfNBmVVdkRERE5Oi1Z2DMOo8/Xk5GSWLl1a7/t07dqVL774orGG1Wi8zckn068D1edjefk0KKtnR0RE5KS0igZlf3WyGwp6HTeNFawdlEVERE6Vwk4TOtkNBb28DcoAFkv13jqgnh0REZGTpbDThBwBVoICrHSKOskGZYdvj47VajEfmzsoK+yIiIg0SKvZVNAfzf31mHr7kmoTVqNnp2a/jvs1TWOJiIicDFV2mpjFYsFisdR/YQ01e3Zq9utAdYNyhdNFeZXz9AcoIiLi5xR2WqFQn8qOb/EttEYQKlR1R0REpF4KO61QXZUdm9ViNjBrKktERKR+CjutUM3qzbE9O1A9laUmZRERkfop7LRCITWmscKDju8hrz75XGFHRESkPgo7rVCY48TTWABhWn4uIiLSYAo7rVBI4IkblKF6k8Gi8spmG5OIiEhbpbDTCoXW0aAMNXZR1jSWiIhIvRR2WqGQOjYVhBo9O5rGEhERqZfCTisUaLNi9xwRERFcS4OyKjsiIiINprDTClksFkI91ZvaKjvVPTsKOyIiIvXR2Vit1E2pXdl4MJ++CeHHvabKjoiISMMp7LRS917U94SvhTnc1Z4ChR0REZF6aRqrDareQVlLz0VEROqjsNMGqWdHRESk4RR22iD17IiIiDScwk4bFK6DQEVERBpMYacN0kGgIiIiDaew0waFe1ZjlVe5qKhytfBoREREWjeFnTYotMZxEsWayhIREamTwk4bZLdZCQ5wBx717YiIiNRNYaeN8q7IUt+OiIhI3RR22ijttSMiItIwCjttVHVlR7soi4iI1EVhp40KU2VHRESkQRR22ijttSMiItIwCjttVHiQe68dVXZERETqprDTRoXrfCwREZEGUdhpo9SzIyIi0jAKO22U9tkRERFpGIWdNqq6sqOl5yIiInVR2GmjzJ4dTWOJiIjUSWGnjTIrO5rGEhERqZPCThtl7rNzEpWdbzZn8smGg001JBERkVbJ3tIDkFNzsg3K5VVOpr+zniqXi3F94ogMCWjK4YmIiLQaquy0URHeTQUbGHYy88uocLpwGZBVWNaUQxMREWlVFHbaKO80Vmmlkyqnq97rD+VVB5wjReVNNi4REZHWRmGnjQp1VM9AFpc7670+I7/U/D6nqKJJxiQiItIaKey0UYF2Kw67+4+voKz+vXYy8qsrOzmq7IiISDuisNOGRYcEApBXUn/YOZRXXdk5osqOiIi0Iwo7bVhMqDvs5BTXX6nxqew04HoRERF/obDThnnDTm5x/ZUaVXZERKS9Uthpw04m7NSs7Gg1loiItCcKO21YQ8NOSUUV+aXVfT1ajSUiIu2Jwk4b5g07R0vqDi8199gBrcYSEZH2RWGnDTMblOup1Hj32EmMDAKguMJJaUX9e/OIiIj4A4WdNqyh01gZnspOn/hwAm3uP3L17YiISHvRomFn5syZjBo1ivDwcOLi4rjiiivYvn27zzVlZWVMmzaN2NhYwsLCmDJlCllZWT7XpKenc8kllxASEkJcXBwPPPAAVVUNPw28rTLDTn3TWJ7KTlJUEB3CvMvV1bcjIiLtQ4uGnaVLlzJt2jRWrVrF/Pnzqays5KKLLqK4uNi8ZsaMGXz22Wd88MEHLF26lEOHDnHVVVeZrzudTi655BIqKipYsWIFb775Jm+88QaPPPJIS9xSszrZyk5iZDCxYQ5AfTsiItJ+2Ou/pOl89dVXPo/feOMN4uLiWLt2Leeeey75+fn861//4p133uH8888H4PXXX6d///6sWrWKM888k2+++YYtW7awYMEC4uPjGTp0KE888QQPPvggjz76KIGBgS1xa83CG3bySyupcrqw22rPrt7KTkJkELGeyo6msUREpL1oVT07+fn5AMTExACwdu1aKisrGT9+vHlNv3796NKlCytXrgRg5cqVpKSkEB8fb14zYcIECgoK2Lx5c62fU15eTkFBgc9XWxQVHACAYUBe6YmPjPDusZMUGUwHT2VHGwuKiEh70WrCjsvl4p577mHMmDEMGjQIgMzMTAIDA4mKivK5Nj4+nszMTPOamkHH+7r3tdrMnDmTyMhI8ys5ObmR76Z52G1WokLcgedEU1mGYZDh2T05Maq6sqO9dkREpL1oNWFn2rRp/PTTT7z77rtN/lkPPfQQ+fn55tf+/fub/DObSn19OwVlVRR7lpknRQbTIdTTs6PzsUREpJ1oFWFn+vTpzJs3j8WLF9O5c2fz+YSEBCoqKsjLy/O5Pisri4SEBPOaY1dneR97rzmWw+EgIiLC56utigmpO+x499iJCgkgONBGh3D17IiISPvSomHHMAymT5/O3LlzWbRoEd27d/d5fcSIEQQEBLBw4ULzue3bt5Oenk5qaioAqampbNq0iezsbPOa+fPnExERwYABA5rnRlpQfZWdmiuxAGK9lR1NY4mISDvRoquxpk2bxjvvvMMnn3xCeHi42WMTGRlJcHAwkZGR3HLLLdx7773ExMQQERHBb37zG1JTUznzzDMBuOiiixgwYAA33HADzz77LJmZmfzhD39g2rRpOByOlry9ZlFf2DH32PHsnly9GkthR0RE2ocWDTuzZ88GYNy4cT7Pv/7669x8880APP/881itVqZMmUJ5eTkTJkzg1VdfNa+12WzMmzePO++8k9TUVEJDQ7npppt4/PHHm+s2WlSDKztR7rDT0bMaK7e4HKfLwGa1NMMoRUREWk6Lhh3DMOq9JigoiFmzZjFr1qwTXtO1a1e++OKLxhxam1Fb2CmvcuJyQXCgzazseKexoj3XuwzIK6kwNxkUERHxVy0aduT0HRt2DMPg8le+48DRUh6c2JeDR6uPigAI8CxXzyupJKdYYUdERPyfwk4bd2zYySooZ1tmIQAPf1K9qaK3sgMQGxpIXkklR4rK6RMf3oyjFRERaX6tYum5nLpjw87WTPdu0JHBAYQE2szrkmqEHe2iLCIi7YkqO21czZPPDcNgu6eqc07vDvzu4n7M/GIbFgskxxwfdnQYqIiItAcKO22cd9+ciioXxRVOtmW4Kzv9EsLpHB3CrKnDj/8ZHRkhIiLtiKax2rjgQBtBAe4/xtyiCrNfp1/CiXeFrp7GUmVHRET8n8KOH/BWd7ILy9h9uAiAvgknbjzWxoIiItKeKOz4gehQ98nnP+w7SqXTIMxhp3N08Amvj9VhoCIi0o4o7PiBGE94WbE7B3BXdSyWE++M3DFcPTsiItJ+KOz4gZgQd2VnTVouUPcUFlRXdtSzIyIi7YHCjh/wVnZKK50A9K8v7Hh6dkoqnBSVVzXt4ERERFqYwo4f8IYXr751rMQCCA8KINFzCvqmA/lNNi4REZHWQGHHD0SHHBN2GnAExPCu0QCsSz/aJGMSERFpLRR2/IB3F2WApMggIj09PHUZ6Qk7P+zNbbJxiYiItAYKO36gZtiprznZa4RZ2cnD5TKaZFwiIiKtgcKOH6gZdvol1t2v49U/MYKgACv5pZXsOVLUVEMTERFpcQo7fiC2ZthpYGUnwGZlSOcoAH7Yq74dERHxXwo7fiAyOACb1b2JYEOnsQBGdnNPZa3dp7AjIiL+S6ee+wGr1cI9F/Qms6CsQSuxvLx9O2u1IktERPyYwo6f+M0FvU/6Z4Z3cYedPYeLyS2u8On9ERER8ReaxmrHokIC6RUXBsA6TWWJiIifUthp50Z00VSWiIj4N4Wdds7s29GKLBER8VMKO+2c99iIDQfyyCooa5LPKK9yYhjauFBERFqGwk4717NjKCmdIqmocnH3u+txNvJuykeKyhn77BKu+8fqRn1fERGRhlLYaecsFgsvXjOUkEAbq/bk8vKinY36/v9ZuY/MgjJW7snhSFF5o763iIhIQyjsCD06hvHklYMAeGnhTlbuzjnums2H8tmVXXhS71tW6eTt1fvMxxsP5J3WOEVERE7FKYWdN998k88//9x8/Nvf/paoqCjOOuss9u3bV8dPSmt15bDO/GxEZ1wG3Pf+Bp/prIKySqbMXsHPX1t5UtNcn/14iCNFFebjDel5jTlkERGRBjmlsPPUU08RHBwMwMqVK5k1axbPPvssHTp0YMaMGY06QGk+j18+kEC7lUP5ZRw8Wmo+vzOriLJKF0dLKsktrqjjHaoZhsG/lqcB0KNDKAAbDuQ3/qBFRETqcUphZ//+/fTq1QuAjz/+mClTpnD77bczc+ZMvv3220YdoDSfkEC7GUx2H64+CX1Pje+zCxu2Ymvlnhy2ZRYSHGDjscsHAvDj/jytyhIRkWZ3SmEnLCyMnBx3X8c333zDhRdeCEBQUBClpaV1/ai0cj061hJ2jhSb32cXNqzJ+N/L9wLwsxGdGd09lkC7lfzSSvbmlDTeYEVERBrglMLOhRdeyK233sqtt97Kjh07mDRpEgCbN2+mW7dujTk+aWY9O7qPj9h9uDrg1KzsHC6oP+xkF5SxcFsWADeP6Uag3crApAgANuzX5oUiItK8TinszJo1i9TUVA4fPsxHH31EbGwsAGvXruXaa69t1AFK86oOO9UBp2bwacjGg5szCjAM6BUXZr7f0OQoAH7cr74dERFpXqd06nlUVBSvvPLKcc8/9thjpz0gaVneaaw9noBT5XSxL6f2aSzDMHhozibiIoK498I+5vM7Mt1L1PsmhJvPecPOhv15TTV0ERGRWp1SZeerr75i+fLl5uNZs2YxdOhQrrvuOo4e1TRFW9bDU4k5UlROfkklB46WUumsbiqu2aCcdqSYd9fs56WFOykoqzSf357lCTvxx4edLYcKKK9yNuUtiIiI+DilsPPAAw9QUFAAwKZNm7jvvvuYNGkSaWlp3HvvvY06QGleYQ47CRFBAOw+UsSeI0U+r9es7ByosTx988EC8/udWe6f6VMj7HSJCSE6JIAKp4ttGSe3OaGIiMjpOKWwk5aWxoABAwD46KOPuPTSS3nqqaeYNWsWX375ZaMOUJpfz7jqqSzvdFZipDsAZRfUHnY2HcwDwOky2Jl9/DSWxWJhiKayRESkBZxS2AkMDKSkxL2EeMGCBVx00UUAxMTEmBUfabt6dKhuUvY2J5/Zw92Efriw3Nwr58DR6mXkmzyVnf25JZRVunDYrXSJCfF53yGdowD3fjsiIiLN5ZQalM8++2zuvfdexowZw/fff897770HwI4dO+jcuXOjDlCaX0/vXjvZReSVuntxzuwRw9z1B6lwusgvrSQqJNCnsvPTQfcqK2+/Tq+4MGxWi8/7Du0SBaiyIyIizeuUKjuvvPIKdrudDz/8kNmzZ9OpUycAvvzySyZOnNioA5Tm1zPOXdnZc6R6GqtfQgSRwQFAdd9OzcpO2pFiCsoqq1di1ejX8RqWHIXF4n7fhixhFxERaQynVNnp0qUL8+bNO+75559//rQHJC3PuyIr7UixefBnj46hxIU7yC+tJLugnD7x4WZlx2IBw3BXd3Zke5qTE44PO1EhgaR0imTjgXy+23WEq4arCigiIk3vlMIOgNPp5OOPP2br1q0ADBw4kMsuuwybzdZog5OWkRgRRHCAjdJK9xLxjuEOwoMCiItwsDO7iOzCMsqrnGaFZ1S3GL5Py3WHnToqOwBjenVg44F8livsiIhIMzmlaaxdu3bRv39/brzxRubMmcOcOXO4/vrrGThwILt3727sMUozs1ot5uaCUN3DExfuWZFVWM6hPPc0VEigjbF9OgKwPj3P3Hm5tsoOwNm9OgDw3a4jOhRURESaxSmFnbvuuouePXuyf/9+1q1bx7p160hPT6d79+7cddddjT1GaQHeYx6gelorLtwBuJefe/t1OkcHM6hTJACLtmVT5TIIc9hJ8ixVP9aIrtE47FayCsp9jqQQERFpKqc0jbV06VJWrVpFTEyM+VxsbCxPP/00Y8aMabTBScupWdnp0cH9fUdv2CksM/t1OkUFk+IJO+VVLgD6xIdhsfiuxPIKCrAxqlsMy3cdYfnOI/SKq70CJCIi0lhOqbLjcDgoLDx+F9yioiICAwNPe1DS8mpWdrzfx0VUT2NVV3ZCiAkNpFNUsHl9nxP063iN8UxlLd+VYz63NaOA7JNYoXWkqJxKp6vB14uISPt1SmHn0ksv5fbbb2f16tUYhoFhGKxatYo77riDyy67rLHHKC3AdxrL27PjruwcLiw3Kzudo90hx1vdgfrDjrdvZ9WeHKqcLj798RAXv/gtU/+5ukF9PPtzS0iduZA7/7v2JO5IRETaq1MKOy+99BI9e/YkNTWVoKAggoKCOOuss+jVqxcvvPBCIw9RWkKPjqHEhgaSGBlE52j3TsjVPTtlNcKO+7WUztVhp+8JmpO9BiRFEBUSQFF5Fa9/t5f7P/gRgJ3ZRezKrr+PZ/OhfCqdBmv26tBZERGp3yn17ERFRfHJJ5+wa9cuc+l5//796dWrV6MOTlpOUICN+feOxWaxmDshe6exiiuc7PDslOyt7Aw6icqOzWrhrJ6xfLEpkye/cP/7492rZ/H2bHrX8/OZ+e7prvzSSvJLK83NDkVERGrT4LBT32nmixcvNr9/7rnnTn1E0mrEhPr2X4U57IQE2iipcFJYVgVUh51hXaKIC3eQEBlEh7D6+7bG9OrAF5syARiYFMElgxN59qvtLN52mNvP7QlAWaWTJ+ZtYWS3aK4cVr0nT1aNk9f355YQWSNoeRWXV3Hf+z9ycUoClw/tdJJ3LiIi/qTBYWf9+vUNuu5Eq3DEP8RHBJF2xH2ERHCAzQxEEUEBLLp/HHarpUH/DpzXNw6H3UpMaCD/umkUZZVOnv1qO2v25lJQVklEUADv/7Cft1ens2Brlm/Yya9uZN6fW+JTVfJatSeHrzZnsudIkcKOiEg71+CwU7Ny01iWLVvGn//8Z9auXUtGRgZz587liiuuMF+/+eabefPNN31+ZsKECXz11Vfm49zcXH7zm9/w2WefYbVamTJlCi+++CJhYWFI4+sY7jDDTufoYJ9gE+Zo+KxoUlQwC+8bS0RwABFB7mmoHh1C2XOkmO92HmHCwAT+vTwNgKyCcsqrnDjs7t25M2us2tpf43yumgrK3AeYZubrDC4RkfbulBqUG0txcTFDhgxh1qxZJ7xm4sSJZGRkmF//+9//fF6fOnUqmzdvZv78+cybN49ly5Zx++23N/XQ2y1vkzJUT2Gdqs7RIWbQATivXxzg7ttZuC2bvTnVQaZmaKl5iGh6bu1hp8gzzVZQVkVphfO0xikiIm3bKZ+N1RguvvhiLr744jqvcTgcJCQk1Pra1q1b+eqrr1izZg0jR44E4OWXX2bSpEn85S9/ISkpqdHH3N55j4yA6pVYjeW8vnH8a3kai7cf9gk6AAfzSuka614Cn1VQs2entNb3KiyvMr/PKiijW4fQWq8TERH/16KVnYZYsmQJcXFx9O3blzvvvJOcnOqN6FauXElUVJQZdADGjx+P1Wpl9erVJ3zP8vJyCgoKfL6kYeIiqis7nU6zsnOsUd2jCQm0cbiwnO/TcrFbLfSJd09Hes/iKiqvoqhGkNlfT2UHfCtBIiLS/rTqsDNx4kTeeustFi5cyDPPPMPSpUu5+OKLcTrd0xKZmZnExcX5/IzdbicmJobMzMwTvu/MmTOJjIw0v5KTk5v0PvxJY05jHctht5m7KwNcOjiRYcnRABzKc1dwjg0uB46W4nIdvxFhzUCUqbAjItKuteqwc80113DZZZeRkpLCFVdcwbx581izZg1Lliw5rfd96KGHyM/PN7/279/fOANuB5pyGgvg/H7V4fWWs3uQ5DmG4qBnE0PvSqzuHUKxWS1UOF1kFR4fZmpWdrJrTHuJiEj706rDzrF69OhBhw4d2LVrFwAJCQlkZ2f7XFNVVUVubu4J+3zA3QcUERHh8yUNU3Maq7ErOwATBibQJSaEy4YkkdI5kqQod7g6lO8JO55gkxQVZJ7HlZ5z/FRWoSo7IiLi0aINyifrwIED5OTkkJiYCEBqaip5eXmsXbuWESNGALBo0SJcLhejR49uyaH6reToECKC7ESGBBAb2viHvsaEBrLst+eZj72B5qBnGisz312lifdUmNJzS9h/tJRj/7TVsyMiIl4tGnaKiorMKg1AWloaGzZsICYmhpiYGB577DGmTJlCQkICu3fv5re//S29evViwoQJgPuIiokTJ3Lbbbfx2muvUVlZyfTp07nmmmu0EquJBAfaWHjfOALt1mbZQNLbBH0orxTDMMzgEh8ZhCPAynfk1Lr8vKhcYUdERNxadBrrhx9+YNiwYQwbNgxwH0kxbNgwHnnkEWw2Gxs3buSyyy6jT58+3HLLLYwYMYJvv/0Wh6N6KuXtt9+mX79+XHDBBUyaNImzzz6bv//97y11S+1Cx3BHs51HlRDpruCUVbo4WlJpBpeEiOoDSg/UG3bUsyMi0p61aGVn3LhxGMbxK2m8vv7663rfIyYmhnfeeacxhyWtiMNuo2O4g8OF5Rw8Wmr238RHOIhxuqfRaqvsFJb59uwYhqGjTERE2qk21aAs7VNSjb4d78qq+IggkmPclZ3ajowoKq80v6+ocpFfWnncNSIi0j4o7Eir19kTdg4cLanu2YkIoosn7GQVlFNWWX0kRKXTRVmlC4AAm7uaoxVZIiLtl8KOtHre5eebDxVQ5TKwWNx9Q9EhAYQGug8HPXC0+tiImiuxajtiQkRE2heFHWn1vNNY69KPAtAhzEGAzb0azJzKqtG3421ODgqwmkvXtSJLRKT9UtiRVs8bdvZ5Ng+Mr7GxYW19O97m5DBHgHltVr7CjohIe6WwI62etzrjlRBRfWSFt2+n5i7K3spOeJDdvLa2IyVERKR9UNiRVu/YsBNXI+wkezYdrFnZ8a7ECnPYzWu9Oy+LiEj7o7AjrV5USADBATbzcc3KTnXPTnWDcvU0VnVlJ1uVHRGRdkthR1o9i8VirsgC37AT7/n+SFF15abmNFa8WdlR2BERaa8UdqRN6OQ5GgJ8T16PDXPvopxbXIHL5d6N27v0PCzITnyk+9ojReVUOV3NNVwREWlFFHakTehUs7ITWf19jOfk9SqXQUGZu1fHrOw47MSGOrBZLbgMOFJU0YwjFhGR1kJhR9qEpMjqJuX48Oqw47DbCA9yH/HmDTOFNSo7NquFuHDP8nPttSMi0i4p7Eib4N1rJ9BuJSrE98T1DmHuMJNb7A473spOmMN9nbkiS2FHRKRdUtiRNqFrrLtnp3NU8HGnl3unsnI8Tco1e3YAEjw9PtkKOyIi7ZK9pQcg0hDDu0TzwIS+DEuOOu61WE/YOXJMZSfc4f7XO16VHRGRdk1hR9oEq9XCtPN61fparGcay1vZKSyv3mcHqsOODgMVEWmfNI0lbV6HMO80lqey41mV5Z3G8oadmkdKiIhI+6GwI22edxorp9jTs3NMZSelUyQWC3y/N5f/rtrXMoMUEZEWo7AjbV71NJa3slO9gzJA34Rw7r+oLwCPfrqZVXtyWmCUIiLSUtSzI21edWWnAqfLoLjCCVRXdgB+Pa4n2zIL+ezHQ9z537VcN7oLP+w9ypaMAv7fuT2Yfn7vFhm7iIg0PVV2pM2r2aBcXFFlPu/t2QH3+VrPThlMSqdIjpZUMmvxblan5VJYVsW7a/Y3+5hFRKT5KOxIm+c9H+toSSX5Je7m5ECbFYfd5nNdcKCNf9w4kvH947hyWCcev3wgAAeOlpobEoqIiP/RNJa0edEhgVgsYBiQnutecVWzqlNTQmQQ/7xplPn49e/2knakmE0H8xnbp2OzjFdERJqXKjvS5tmsFmJC3NWdvTnFgG+/Tl1SOkUCsOlAXpOMTUREWp7CjvgF71TWPs9eOg0NO4M7e8LOwfymGZiIiLQ4hR3xC97zsfYe8VR2TjCNdazqyo7CjoiIv1LYEb/gXZHlreyEN7CyM9Cz4eCh/DIOF+o4CRERf6SwI36hg6eysy/35Co7YQ47PTuGAfBTM01lbTqQz4XPLeWbzZnN8nkiIu2dwo74BW9lp6zSBVTvntwQgz1TWRubaSrrs42H2JldxIdrDzTL54mItHcKO+IXvA3KXmGOgAb/bIrZpJzXmEM6od3ZRQCkefqLRESkaSnsiF+IDXX4PD6pyk7n5q3s7D7sDjv7ckpwuoxm+UwRkfZMYUf8wvGVnYaHnQGJkVgtkF1YTlZBWZ3XllU6MYxTDyhllU5z48MKp4tDeaWn/F4iItIwCjviF7yHgXqdTNgJDrTRJz4cqLu6sz79KAP/+DVPf7Xt1AaJu5pTs5izR1NZIiJNTmFH/IK3QdmroauxvLz77czbeIjPN2Ywb+Mh8kp8z8v67McMnC6D15fvJbuw7grQiezy9Ot4pR0uOsGVIiLSWBR2xC9EBNkJsFnMxw3dZ8fL27fzyYZDTHtnHdPfWc9vP9zoc833e3MA9/TTWyv2ndI4dx8TbtSkLCLS9BR2xC9YLBafJuWTrexcOjiJ8/vFMbxLFCO6RgOwdMdhyiqdABSUVbLlUIF5/X9W7aOkouqkx+kNOz06hgKaxhIRaQ4KO+I3ajYpn0zPDkB0aCD/vnkUc349hg/vSCUxMojyKher03IBWLvvKC4DkmOC6RobQn5p5Sntk+OdxrqwfzwAew4r7IiINDWFHfEbMTWalE+2slOTxWJhbJ+OACzdfhiANZ7Qc2b3WG49uzsA//w2jSqniw3783ht6W7SPUdVnIjLZZjh5sIB7rBzKL/UrB6JiEjTUNgRv9GhRpNy+ElsKlgbb9hZsiMbgO89YeeM7jH8bEQy0SEBpOeWcObMhVwx6zue/nIbv3pzDeVVJw4uh/JLKa10EmCzMDQ5ivAgO4ZRfZ6XiIg0DYUd8Rve5ec2q4WggNP7V/usXh2wWS3sOVzMruxCfjyQB7jDTnCgjRvO7ArAkaIKQgJthDvs7Mou4tXFu0/4nrs9VZ1usaHYbVZ6dHD37aQd0YosEZGmdOq1fpFWxrv8PMxhx2Kx1HN13SKDAxjRJZrv9+bywoKdVDoN4iMcdIkJAeDX5/UiONBO19gQzusbx8JtWUx/Zz2vLtnFJYMTzX17avL26/SKcx882r1DKD8eyFeTsohIE1NlR/yGt0H5ZJuTT2RsX/dU1ryNGQCc0T3WDFFBATbuHNeTSSmJBAfauCQlkfH946h0Gjz40cZaj4HwrsTynrLevYP7f9PUpCwi0qQUdsRvdAx3V3Yig0+vX8fL27fjdUb3mBNea7FYeOKKQYQ57KxPz+OGf63mlUU7WbHrCC5P8PEeANozzj191b2jdxpLYUdEpClpGkv8xlk9Y5k6ugvj+sY1yvsNSIygQ1ggR4rcOymPriPsACRGBvP7S/rz0JxNrNidw4rd7k0Ix/ePY/b1I8zKTq+O7imu6p4dhR0RkaaksCN+w2G38eSVKY32flarhXP7dGTOuoNEhwTQyzP9VJdrz+jC4M6RrN6Ty/r9eXyzOZMFW7P59dvrzNDk3VCwuyfs5BRXkF9SSWRI41SkRETEl6axROpw6eBEAC7oH4/V2rCm54FJkfzq7O68fO0wXrt+BHarhflbsgBIigwi1NNTFOqwEx/hnnpLy1F1R0SkqSjsiNTh/H7xfHHXOTx22cBT+vnz+sXx16uH4F0c1jPOtzrUXcvPRUSanMKOSD0GJEWY1ZhTcfnQTjx5RQoOu9XcOdlLK7JERJqeenZEmsF1o7vw85GdCbD5/vdFT0//zs5sVXZERJqKKjsizeTYoAOYmw9uzyz0eb7S6WLRtqw6j58QEZGGUdgRaUH9EtxhZ29Osc+BoK9/l8av3viBmV9sO633NwyDI0Xlp/UeIiJtXYuGnWXLljF58mSSkpKwWCx8/PHHPq8bhsEjjzxCYmIiwcHBjB8/np07d/pck5uby9SpU4mIiCAqKopbbrmFoiJNCUjb0DHcQXRIAC4DdmZV/3u70rNHz0drD1BacerVnde/28vIPy3gi00Zpz1WEZG2qkXDTnFxMUOGDGHWrFm1vv7ss8/y0ksv8dprr7F69WpCQ0OZMGECZWVl5jVTp05l8+bNzJ8/n3nz5rFs2TJuv/325roFkdNisVjo66nubMssANwhf+OBfAAKy6tOK6h4f9a79F1EpD1q0Qbliy++mIsvvrjW1wzD4IUXXuAPf/gDl19+OQBvvfUW8fHxfPzxx1xzzTVs3bqVr776ijVr1jBy5EgAXn75ZSZNmsRf/vIXkpKSmu1eRE5Vv4QIVu3JNft2DuWXkVNcYb7+3g/7mTKi80m/b0WVi40H3aHpJ8//ehmGgdNlYK+lj0hExN+02t90aWlpZGZmMn78ePO5yMhIRo8ezcqVKwFYuXIlUVFRZtABGD9+PFarldWrV5/wvcvLyykoKPD5Emkp3srO9ix32Nl0IA9wb0BoscD3abmndKTE5kP5VFS5APchpCUVVeZr/1m1j16//5LlO4+c5uhFRFq/Vht2MjMzAYiP992XJD4+3nwtMzOTuDjfc5DsdjsxMTHmNbWZOXMmkZGR5ldycnIjj16k4aqnsdxh50fPFNbYvh05t7f7MNL3f9h/0u+7Lj3P/N5lwNaM6lD/wQ8HAPhqs3p5RMT/tdqw05Qeeugh8vPzza/9+0/+LxKRxuJdfn64sJzc4go2ecLO4M5R/GKUO4h/tPYAVU7XSb3vuvSjPo9/OugOOwVllWw+5P6MbRmFx/2ciIi/abVhJyEhAYCsLN/GyqysLPO1hIQEsrOzfV6vqqoiNzfXvKY2DoeDiIgIny+RlhLmsJMcEwzAtowCNnqmsVI6RTK+fzwxoYFkF5bzyuJd5JzEMvL1+9xh54xu7tPavX07a/cdxWW4r9mWWYhhGI10JyIirVOrDTvdu3cnISGBhQsXms8VFBSwevVqUlNTAUhNTSUvL4+1a9ea1yxatAiXy8Xo0aObfcwip6qvp7rzzZYsCsqqCLRb6ZsQTqDdys9HupuTX1iwkzOeWsj1/1zN15szcXkSS5XTxUdrD3DvexvY6+ntycgv5VB+GTarhetGdwHgp0Puys7qPbnm5xaVV3HgaGmz3aeISEto0dVYRUVF7Nq1y3yclpbGhg0biImJoUuXLtxzzz386U9/onfv3nTv3p2HH36YpKQkrrjiCgD69+/PxIkTue2223jttdeorKxk+vTpXHPNNVqJJW1K34RwFmzN5uMNBwEYkBhh7rh8/0V9iQkJ5LONh/jpYAHLdx1h+a4j9OwYyiWDk/h4/UHSc0sAyCwo453bzmTdvjzAvWnhGd3dlZ2dWYWUVTpZnZbj89lbMwpIjglppjsVEWl+LVrZ+eGHHxg2bBjDhg0D4N5772XYsGE88sgjAPz2t7/lN7/5DbfffjujRo2iqKiIr776iqCgIPM93n77bfr168cFF1zApEmTOPvss/n73//eIvcjcqr6JrinUvNKKgEY0jnSfC3AZuX/je3JvN+cw9IHxvHrcT0JD7Kz+3AxLy3cSXpuCTGhgQTYLKzYncPK3Tlmv87wLtEkRgYRExpIlctgfXqe2RPknd7aVuOoivIqJ+vTj9Y5tfXVTxlc/beVHDha0rj/EEREmkiLVnbGjRtX5y9Vi8XC448/zuOPP37Ca2JiYnjnnXeaYngizcZ7bIRXSueoWq/rGhvKbyf2485xPXlndTordudwTu8OXDe6CzO/2MZ/Vu3j+fk7qPA0M4/oGo3FYmFgUgTf7jzCWyv3UuUySIoM4sIB8Xy/N9fczBDgyc+38tbKfcyeOpyLUxJrHcO/l+/l+725fLEpg9vP7dk4/wBERJpQq+3ZEWlPuncIJcBmMR/XrOzUJjwogP83tidv/uoMbj2nByGBdn59Xk8C7Va+35vLhv15gLuyAzCok/v9vt7s3pJhdI9Y+ie6q0lbPSuyqpwuPv3xEACbjtmE0MswDHZku6/fc/jk9/4REWkJCjsirUCAzUrPjmEAhATa6OH5/mQkRgZz3RldzMcdwgLNVV6Dktxhx7sKa3T3GPolVh9CWlJRxQ/7jprTaJkFZdTmcGG5eY3Cjoi0FQo7Iq2EdyprUKdIbFZLPVfX7tfjeuKwu/9vPbyLewrL/Z6+2yuM7hFLhzAHHcIcGAbsyCpiQY3zs7JOEHZ21DisdM8RHbgrIm2Dwo5IK3FuH/duyRf0i6vnyhOLiwji9nN7ADB+QPXu411iQggPcrfoxYU76BbrXn3V31Pd2ZpRwPyt1WEnM7/2sOM90gLgSFEF+aWVpzxWEZHm0qINyiJS7cphnRjWJZoup7kM/N4L+/CzEZ193sdisTAoKZKVe3I4o3uMWfHplxDOtzuP8NmPh9iXU726Kqug9s0Ld2b57ri853ARwzx9QSIirZUqOyKthMVioXuH0FOewqr5Pl1jQ81A4zVpsHt11ZTh1Seo9/MseV+x2733jnc5elF5FUXlVRzLW9nxvnXNvp256w9w1avfkZGvTQpFpHVR2BFpJ64f3YXtf5rIeTWmybwrsrwuH5ZEuMNd8D12KsswDHZ6enZGdnVXc2qexv7q4t2sS8/jQ88hoyIirYXCjkg7YbFYcNhtPs/1jAvFXqOSNL5/PPGR7k07j21SPpRfRlF5FXarhQv6u/uBvE3KR4sr2Jnt/n51Wq7Pz/17eRrjn1vKoTxVfESkZSjsiLRjDrvNXPI+pHMk8RFBJES4w86xlZ0dnimsHh1DzbO8vNNYa/dVn7C+dt9RKj2bGrpcBq8u2cWu7CK+/CmzaW9GROQEFHZE2rmhyVEATBiUAEC8J+xkFfqGHW9zcu/4cHp0DAXc01gul8GavdXVnNJKp7kp4aaD+RwpqgBg86HaNyo8FZsP5TN3vabLRKRhtBpLpJ377cS+jOwWzRXDOgEQH+EAIOuYys72TPc0VZ+4cDpHhxBos1Je5eJgXqkZdhx293Or9+QyvEs0i7dnmz+/5VABjWXGexvYkVVEUmQwo3vENtr7ioh/UmVHpJ2LDXPw85HJ5inrCZ6enWN3Ud7pOSaib0IYNquFrp69erZkFJiVnGs9Ozh7T1ZfvP1wjZ8voqzSedrjLamoMvuD1qYfredqERGFHRE5hncaK7PGXjsuV/VKrN6efh3vVNbH6w9S6TSIC3fwsxHuZe0/7D1KdkEZGw/kAe4jMJwuw+z7OR27sovwnh/sPcFdRKQuCjsi4sPboFxzGuvA0VJKK50E2q109WxW6D2/a77nmIlR3WLonxhBeJCdovIqXl2yG8NwH1XhPZC0MaaytmdWB6aNCjsi0gAKOyLiwzuNdbioHKfn5FDvZoI9O4Zh90x3de/gruxUea4Z2S0am9XCKM/GhG+v3gfAeX3jGJDk3s9ncyOEnZrVoYN5pRwpqn23ZxERL4UdEfHRIcyBzWrB6TLMIOENGH3jq09j7+mZxvLyhpzR3d3/W+l0h6BxfeMYaIad2isxLpdBScXxOzbXZnuW7wGkmsoSkfoo7IiID5vVQscw94os71472zKrl5179ehQHXzCHHbz1Paaq6OiQwIYmhxlhp2tGYVmtQjcx1L8e3ka5/55McOfmM+KXUfqHd8Oz1i8h5lqKktE6qOwIyLH8S4/zywowzAMVu9xr64a5tmTByA6NJDokAD3812izOmtgUkRhAS6d2oe26cjNquF7h3CCA6wUVrpNI+Y+Hj9QVJnLuTxeVs4cLSUskoXv/nf+jrP1sovqTRXiXnP+PI2QYuInIjCjogcx9xYsKCMXdlFZBeW47BbGd7V94Rzb5OydwoLIMBm5dzeHQGYlOI+fNRmtdAv0V352Xwon+yCMv5v7iYKy6ro0SGUP10xiAGJEeQUVzDt7XVUVLlqHZe3d6hTVDBn9XJXkDYezMcwjFqvFxEBhR0RqYW5105+Gd95ppZGdYshKMD3bK3bzunBWT1juXpkss/zM69K4d3bz+SigQnmc96prC0ZBbywcCclFU6GJkex4N6xXH9mV167fgQRQXbWpefx1Bdbax2XN+z0TQhnQGIkNquFw4XlZBWoSVlETkxhR0SOU73XThnf7XZPYXkrKTVNHJTAO7edaYYjr+jQQM48ZmfjgUmRAMzfnMV7a/YD8H+T+mP1HETaJTaE564eCsAbK/Zy1//Wk3PMSitvv06f+HCCA230jnNXln7UVJaI1EFhR0SO491r51BeKas8YefsXh1O6z29lZ09R4pxugwuHBDPGd1jfK4ZPyCe/5vUD6sFPv3xEOOfW8qnPx4yX6+u7HgPL40CtCJLROqmsCMix/FWatbuO0pheRURQXazMnOq+sSHY/NUcWxWCw9O7Ffrdbef25OPp42hX0I4R0squet/65m/JQvDMMwNBft4VoWldHaPSZUdEamLwo6IHMc7jeXdK+esnh3MoHKqggKqp52uHplMr7iwE147uHMUn04/m2tGuXuBnpi3hf25peSXVmK1uDc3hBqVHTUpi0gdFHZE5DjepedeY2rp1zkV91/UlyuHdeKBCX3rvTbQbuXhSwcQH+EgPbeEBz/aCEC3DqFmo3TfhHACbVbySirZl1PSKGMUEf+jsCMixwkPCiA0sHrl1Vmn2a/jNX5APM//YigxoYENuj7UYef/JvUHYKVnr5++NTY2DLRbGeFZDv/h2gONMsbTYRgGM97bwB3/WeuzeaKItCyFHRGpVbynbycxMogeHULrubrpXDYkiZE19vfpUyPsANx0VjcA/rt6X71HThiGwYMfbuSqV78jv6Sy0cd6MK+UuesP8tXmTPURibQiCjsiUivviqyzenbAYjm9fp3TYbFYePSygXiH0DfBN+xcOCCeLjEh5JVU8tG6g3W+1+ebMnjvh/2sS8/jxYU7G32sPx2sXhW2ZFt2o7+/iJwahR0RqdVZPWOxWOCq4Z1aeigM6hTJ7yf1Z1zfjozt09HnNZvVwq/GdAPg38vTcJ1g+qiwrJLHP9tiPn5r5V52Hy6q9dpTtalG2Fm8/XCjvreInDqFHRGp1a/H9WLDIxcxppH6dU7Xref04I1fnkGow37caz8fmUxEkJ20I8UsPEFF5YUFO8kuLKdrbAjn9ulIlcvgqc9r36m5poz8UvM8r/psOlhQ4/t8sgvLGvRzItK0FHZEpFZWq4XI4ICWHkaDhDrsXDe6KwD//HbPca9vOVTAGyv2AvDYZQP54+QB2K0WFm7LZtmOE1dgXC6Dn7+2kktf+rbe4GIYhjmNFe4JZEtV3RFpFRR2RMQv3HxWN+xWC6vTclmwJct8vrzKyUNzNuJ0GUxKSWBc3zh6dgwzG5ufmLflhAeP7jlSzIGjpRRXOFm+80idn38ov4zc4gpsVgvXndkFgCUKOyKtgsKOiPiFhMggM8Dc+/4G0nNKMAyD38/9iR8P5BMRZOfhSweY1991fm+iQwLYmV3EX7/ZXut7bjqYZ37/3a6cOj/fW9XpHRfGRM8BqMt2HqbSWXuQ8vpmc2ad1SUROX0KOyLiNx6c2I9hXaIoKKvizrfXMnvpbj5cewCrBV65bjiJkcHmtZEhAcy8ajAAf1u2h8W19Pr8uL+64fi7XUfq3KXZG3ZSOkUyuHMUMaGBFJZVsW7f0RP+zOZD+dz+n7X86o01ZBeov0ekqSjsiIjfCLRbmXXdcGJCA9l8qIBnv3JXbH5/yQDOPWYVF7hPbb8p1d3rc+/7G8jM9w0cG2vslZNZUMaeOhqVvSuxUjpHYrNazFVj87dkMW/jIW7412rufW+Dz2aDry119xdVuQw+XNfymyKK+CuFHRHxK0lRwbx4zVBzX56rR3Y2l6bX5qFJ/RmYFOE+dPTd9ebS9Sqni82H3KurOkW5K0IrdlX37ezIKjTDUM3m5EGd3IeTjuvrDjv/XJ7G9HfW8+3OI8xZf5A3PY3S6TklfL6x+kT399fs1/leIk1EYUdE/M45vTsye+oI7r6gN09cMajOTRGDAmy8ct1wQgJtfJ+Wy+q0XAB2ZBVRXuUi3GHnF54DSZd7wk5mfhlXzPqOK2Z9x/r0o2QWlHGkqAKrBfonRABwbu+OBNrdv2I7hAWafTx/+WY7B/NK+fu3u3EZMLp7DKGBNvbmlJifLSKNS2FHRPzSxEEJzLiwDw67rd5ru3cI5dLBiQB8vsldbfFWbVI6R3J2b/deQyt35+B0GTw3fzslFU5cBvz2w42s2+e+tndcOMGeM8WiQwP5322jee36EXz3u/N5depwRnaNpqTCyX3vb+CDH9zTVveM78NlQ5MAd3VHRBqfwo6ICHDJYHfg+HJTJlVOFxtr9OAM7hRJuMNOQVkVH607wAeeQ0cjguzszC7ij59uBqqnsLxGdI1h4qAEHHYbVquFmVelEGCzsGpPLuVVLoYmR3FmjxiuHumuHH2+KYP80sY/s0ukvVPYERHBfTxGdEgAOcUVrNqTa1Z2hnSOwm6zMrpHLAB/+PgnDAMuSUnk2Z8NAeBIUTkAKZ0i6vyM3vHh3Dm2p/n4znE9sVgsDE2Ook98GOVVLv73fTpf/ZTBI5/8xDur05vgTkXan+P3XRcRaYcCbFYmDkrgf9/vZ866A2zLKARgcGd3tWZMr1gWbM2iospFgM3Cbyf2pWtsKJOHJPHZj+6pr5TOkSd8f69fn9eLHw/kExxg48L+8YD7sNOrRybzp8+38vSX23yuH941in6ePiDDMHh3zX4SIoM4r29co927iL9TZUdExONSz1TWxxsOUuUyiA0NNFdinV3jjLDrz+xK19hQwH38RFJkEHHhDgYk1h92ggJsvPmrM3jthhFYrdWN01cN70xUiPt4jh4dQ+kb7z7d/aUap7N/tjGDh+ZsYvrb66iqZ7NCEammyo6IiMfo7jF0CAvkSFEF4K7qeFdy9YoLY2hyFEeKyrnr/N7mz8SEBvL1jHMBzObkUxETGsj8GWOpcLroFBXM9sxCJrywjC82ZbI9s5CkqCD+NM99antxhZO9OSX0igtr8Pvvzy3hZ6+tYEBiBP+8aRQ264lXqIn4G1V2REQ87J6pLK+UzlHm9xaLhbm/PovF948jOjTQ5+fCgwIIDzr9Q1M7hjvMSlLfhHAmpbjH8tLCnTw/331qu9e2zIJa36M2hmHw4EcbySooZ/H2w7y8aGf9PyTiRxR2RERq8E5lAQw5pgfHYrEQYGu+X5t3XeCuIH3xUwZvrtwLYFZztmcWNvh93vk+nRW7c7B7qjkvLdzJ6j11n/XVEBn5pZz3lyU889W2+i8WaUEKOyIiNYzqFkPvuDAigwMY0TW6RcfSLyGCiwclYBiYp7ZfP9p9ovrWjIaFnQNHS3jq860A/N+k/kwZ3hmXAXe/u4GjxRWnNb456w6SdqSY2Ut213sqvEhLUtgREanBZrXw4Z1nsei+sUSFBNb/A03srgt6Y7VAaKCNhy8dQL9E98qs7Vn1T2MZhsFDczZRXOFkZNdobj6rG49fPpAeHULJLCjjDx//dFpjW1Tj8NTfzdlISUXVab2fSFNR2BEROUZkcACxYY6WHgYA/RMj+PDOs5g7bQyJkcH0S3Cv0tqfW0phWd0bEH63K4dvdx7BYbfy7M8GY7VaCHXYeenaYVgs7k0Md2b5Vohcroadz5VbXMG6dPeJ7h3DHRw4Wspfvt5xCnco0vQUdkREWrnhXaLp41mKHhUSSEJEEOA+jLQub6/eB8AvRiXTo2P1yq1BnSKZMMDd/Pz3ZXvM53dlFzLqyQXc+/6Gese0ZHs2huEOY3/5uXtzxddXpJkBSKQ1UdgREWlj+iW6g8+2OpqUswrK+GZLFgBTR3c97vXbzu0BuPcUyi4ow+Uy+O2HG8kpruDj9QfJLiyrcwwLPVNYF/SLY2yfjkwZ3hnDgGe+VLOytD4KOyIibUxfz1TWtjqalN9bsx+ny2BUt2jz+ppGdI1mZNdoKp0Gr6/Yy39X72Ndeh4ALsN9RtiJVDpdLNt+GIDz+7t3cp5xoXvl2A/7jpJfovO9pHVR2BERaWP6e46PONHy8yqn+4wtqL2q43W7p7rz31X7ePar7UD18RjeIzBqs2ZvLoXlVcSGBjLEsxdR5+gQeseF4XQZfLvr8MndkEgTa9Vh59FHH8Visfh89evXz3y9rKyMadOmERsbS1hYGFOmTCErK6sFRywi0vS8lZqtmQUYhkGV08Vz32znlUU7yS+pZMn2w2TklxEdEuCzSeKxxvePp0fHUArLqigqr2J4lyj+dsMILBZ3heZgXingXvY+Z90B1uzNBWDRVvcU1ri+cT47MY/r2xGAJdsVdqR1afXHRQwcOJAFCxaYj+326iHPmDGDzz//nA8++IDIyEimT5/OVVddxXfffdcSQxURaRY9O4Zht1ooLKviUH4ZX/2UyUuLdgHwt2V7iPHs8PzzkckEBZz4CAur1cJt5/TgoTmbCLBZeGbKYBIjgzmjWwyr03L5fOMhbj+3J68u3sVf57tXWo3t05Fd2UUAXNDf9zDScX3j+Me3aSzZfhiXy/A5+8ursKyS0EB7ra/Vp8rpwt6MmzqK/2j1Ycdut5OQcPx/meTn5/Ovf/2Ld955h/PPPx+A119/nf79+7Nq1SrOPPPME75neXk55eXV264XFDR823URkZYWaLfSKy6MbZmFLNmezXPfuKeg4iMcZBWUU1jm3u/mujO61PtePxvRmfTcElI6RdLbs+Jr8pAkVqflMm9jBiO6xvCC5zBSqwWW7nBXbexWC+f07uDzXiO7RRMaaONIUTlbMgoY1Ml3B+pF27L4f/9Zy9Ujk3nyypQG3+++nGKen7+DT388xG/O782MC/s0+GdFoJVPYwHs3LmTpKQkevTowdSpU0lPd89Dr127lsrKSsaPH29e269fP7p06cLKlSvrfM+ZM2cSGRlpfiUnJzfpPYiINDbvVNYT87ZQXOFkeJcovnvwfF6dOpzR3WO4c1xPunUIrfd9AmxWHpzYj0kpieZzFw9KwGa1sPFAPnf8dy1Ol8EVQ5NYfP84Lh/qPk7j0sGJx50H5rDbOMtzOvziGhsOAhwpKueBDzZS6TR4b81+sgrqXu0FUFJRxe/nbuKCvy7l4w2HcBnwyuJdJ3UumAi08rAzevRo3njjDb766itmz55NWloa55xzDoWFhWRmZhIYGEhUVJTPz8THx5OZeeJVBAAPPfQQ+fn55tf+/fub8C5ERBpfP0+TclmlC5vVwpNXpmC3WZmUksh7/y+VByf2q+cdTiw2zMFZPWMBOFxYTpeYEJ64YhBdY0N58ZphbHz0Iv7s2VvnWGbfzo7qvh3DMPjdR5vI8RxPUeUy+O+qffWO49FPN/P26nSqXAbj+nbk7F4dcLoMHvlkM4bRsM0PRaCVh52LL76Yn//85wwePJgJEybwxRdfkJeXx/vvv39a7+twOIiIiPD5EhFpS7x77QDccnZ3+ic27u+xyUPcFRy71cKL1wz1qeJEBAWc8EDUcX3dfTzr04+SV+IONx/8cIAFW7MIsFm46/xeALyzOp2ySucJP3/x9mze/+EAFgv866aRvPHLM3jmZ4MJCrDyfVoun9axWkzkWK2+Z6emqKgo+vTpw65du7jwwgupqKggLy/Pp7qTlZVVa4+PiIg/Gdo5inCHndiwQO72nI7emC4fmsRPB/MZ1S2GYV0afiBqp6hg+sSHsSOriH9/t5cqp4s3V+wF4L6L+nLr2d35cO0BDuWXMW9jBj8b0fm498gvreShjzYB8MuzunNB/3jzvaef14u/fLODJz/fSlmlk2U7j7D5YD73XdTXDGgix2rVlZ1jFRUVsXv3bhITExkxYgQBAQEsXLjQfH379u2kp6eTmpragqMUEWl60aGBLH5gHJ/95mxCHY3/360Ou43HLx90SgHCW915aeFOXl2ym+IKJ2f2iOG2c3pgt1m5IbUbAK9/l1brdNQT87aQWVBG9w6hPDChr89rt57Tg66xIWQXlvPgR5v4fGMGe3NKmPnFVqqcrpO/UWkXWnXYuf/++1m6dCl79+5lxYoVXHnlldhsNq699loiIyO55ZZbuPfee1m8eDFr167ll7/8JampqXWuxBIR8RcdwhzHNQm3BlcM7USAzUJEkJ3LhiTxwi+G8sYvzzD35LlmVDIOu5XNhwr4YZ/vWVrfbM7kw7Xu6as//2wwwYG+S+eDAmzMvCqFqJAABneO5K7ze9EhLNC9BH9z3f2a4O4f2plVeFzIKi6vYu76A/UekyFtU6uexjpw4ADXXnstOTk5dOzYkbPPPptVq1bRsaO7Ae7555/HarUyZcoUysvLmTBhAq+++moLj1pEpH0bkBTBxj9OwG6z1NrbEx0ayJXDOvHumv08+flW3r51NKEOO/tzS7j/gx8BuO2cHozsFlPr+5/VswMbHrnIfGyxWHhx4U7+vTyNSwfXXYl6Y8VeHvtsC1cN78Rffz4Ei8WC02Vwx3/X8u3OI4QE2rhjbE9uO6fHcUGrKezPLWH+lix+MSq5SSp04mYx1NJOQUEBkZGR5Ofnq1lZRKQZ7D1SzORXllNYVsWZPWL42/Ujuf5fq9l0MJ9hXaJ47/ZUAu0Nm3zILizj7KcXU+F0MffXZzGsSzTlVU7W7j3KyG4x5vs4XQbnPrvY3Bn6/yb14/Zze/LcN9vNTRm9EiKC+NsNIxiSHFXrZ85Zd4D/rtrH45cPOm4/oZNx+1s/8M2WLC4cEM/frh9xSpsttmcN/fu7VU9jiYiIf+rWIZS3fnUGYQ47q/bkMu4vi9l0MJ/okABmXTe8wUEHIC48yOwt+vd3e9mXU8yU2Su47p+rzUoRuPf+OZhXit0TKJ7+chtPf7nNDDrP/2IIL187jM7RwWQWlHH3u+sprfBdMeZyGTz95Tbuff9H1qXn8bdle+ocm9N14nqCYRjmNN78LVm8umTXCa+V06OwIyIiLWJYl2je+OUoQgJtHC2pxGKB538xlKSo4JN+r1+d3Q2ALzZlcOlLy/npoHvjwU9/PMRaT6D4j2dvn1+d3Z1rRiXjMuC1pbsBuDG1K1cO68zkIUl8cfc5JEQEsTenhBcW7DA/o7i8ijv+u9b8GYBFW7OOW0JvGAYrdh/hF39bSb+Hv2T+ltrPbEzPLSHXs/cQwF/n7zB3qK6p0uliX04x5VUnXqovdVPYERGRFjOyWwz/vnkU/RLCeeTSAeZKrpM1MCmSM3vE4HQZFJZXMbJrNBd7DkF9Yt4W9h4pZumOw1gsMHV0Fx6/fBAju7qX1A9NjuL3l/Q33ysiKIA/XTEIgH98u4eNB/LYfCifya8s55stWQTarDx39RASIoIornCyfOcR82f3HinmF39bxXX/WM3qtFwqnQYPf/wTJRVVx415fXoeAMO6RHHtGckYBtz1v/XmNJvXI59sZuyflzDwka+56Pml3Pv+Btbuyz2lf07tlcKOiIi0qDN7xPLVPefyyzHdT+t9HpzYj54dQ5l2Xk/+d/uZPHbZQEICbWzYn8cd/10LuA8y7RobSqDdyr9/OYpnfzaYN395Bg67bzPy+AHxTB6ShMuAO/6zlitnrWDP4WISIoL43+2juWp4Z/NE+S9+ygDcFZ1739/A93tzCbRZuTG1K8kx7imxWYuPn6Jal+6uOA1LjubRywYyuHMk+aWVvOXZlwiqV4mBe+fpHVlFzFl3kCmzV/Kz2SuOO5ZDaqewIyIifmFYl2gW3jeOByb0I8BmJS4iiDvH9gRgW2YhADec2dW8PiIogKtHJhMZUvvy/T9OHkB0SACH8suocLq4cEA8X959DiO6uleJeStHC7ZkUVHlYtnOI6xLz8NhtzL/3nN5/PJB/OGSAQD8Y1ka+3KKfd7fW9kZ3jUKh93Gr8e5x/rxhoNmr8+CrVmUVbroGhvCit+dz+s3j+IXI5MJtFn5Yd9RfvnGmhNOk0k1hR0REfFbt57Tg8TIIAA6Rwef1DRZhzAHz109lAGJETxx+UD+fsMIokMDzddHdouhQ1ggBWVVrNyTw/Pz3f0915/Zla6x7kNYLxoQzzm9O1DhdPHEvC3mz5ZWONma4e4r8u5QfV6/OCKDA8gqKGfl7hwAPvvRXTWaPDiJpKhgzusXxzM/G8y3D57HpYPdh7e+sniXzgqrh8KOiIj4reBA907Q4Q47M8b3MTc2bKjz+sXxxd3ncENqNywW35+1WS1MGOiu7vxp3hY27M8jKMDKHZ5qErj3APrj5AHYrRYWbM1mmacB+adD+VS5DOLCHSR5wpjDbjMDzJz1B8gvqWTpDvc01WVDffcPio8I4tHLBuKwW/lxfx7fp51cD8//zd3E6KcWHFdtqmnFriP889s9frEztcKOiIj4tQsHxLPpsQlMqeUcrtN18SB3ONmZXQS4p8k6hjt8rukVF86NniMynpu/A8MwWO/t1+kS5ROirhreCYCvfspk7voDVDoN+saH0yc+nGN1CHOY9/T3Gkvg1+47yksLd5KRX3rczwCs3J3DO6vTySooZ/aS3bVe88PeXG5+fQ1/+nwr//4urd5/Dq2dwo6IiMgpGt0jhihPz09wgI3/V6OqU9Od43risFvZsD+P5buO1FiJ5XvI6vAu0XSNDaGkwsmfv94OwOQhiSf8/NvO6YHFAgu3ZbMzq5CvN2dy7d9X8dz8HYz98xKemLeFI0Xl5vVOl+EznTZn3UGyC3yPyNifW8L/+89aKjwVnb9+s4O9R05cAWoLFHZEREROUYDNak493TymGx3CHLVe1zHcwdTR7uboFxfsrA47x+zQbLFYuGKou7pT7NnQsK7DWLt3COWiAe5T4e95bwO/fnsdFU4X8REOKqpc/Gt5GmOfXWyu6Ppw7X62ZBQQHmRnYFIEFU4Xr9dY/VVYVsmtb/5ATnEFA5MiSO0RS3mViwc/2oirng0SH/tsMxNfWMaM9zbw7+VpfLj2ADO/3Mqv3ljDOc8uqnX5fXPRcRHouAgRETl1xeVVrNidw3l9O2Kv5Swwr6yCMs55djEVVe6Kic1qYdOjFxES6Hsm1t4jxYz7yxIAhnSO5JPpZ9f5+Wv3HWXK7BXm4ynDO/PMlBRW7M7hL99sZ+OBfACuHtmZRdsOc6SonD9c0p8uMSHc/p+1hAfZWfnQBTid7jPCVu7JoWO4g0+nj6GyymDCC8sorXTy5JWDzMB2rDnrDnDv+z/W+prXp9PHMLhzVJ3XnKyG/v2tU8dEREROQ6jDzoWe6kpd4iOCuHZUMm+udO/k3D8x/LigA+6jNEZ2jeaHfUfrrOp4jegazejuMaxOy+XG1K48OnkgVquFc/t0ZEyvDry8aCcvLtzJ+z+4qzvdO4RyY2o37FYLPTqGsudwMc9+tY1lOw6zN6eEkEAb/7hxJImR7p2s75/QlyfmbWHmF9u4oF88CZ6Gaq9DeaX88dPNgLtnqUOYg00H8ympqKJXXBi948PpHRdGr7iweu+lqaiygyo7IiLSPDLySxn77BIqnC5uOLMrT3h2aj5Wek4Ji7ZlMfXMrrWeHH+svJIKdmUXMaJr9HGrxgC+23WEu9/dQE5xOf+8cSQX9HeHs/fWpPPgR5vM6zpFBfOPG0cyIKn670Kny2DK7BVs2J/HJSmJzJo63HzN5TK48d/fs3zXEYZ1ieKD/5daZ3WrsekgUBERkVYmMTKYW89x7xTt3ZSwNl1iQ7h5TPcGBR2AqJBARnaLqTXoAIzp1YFF949l/oyxZtABuGJYJ+Ij3H1GZ3SL4ZPpY3yCDrin2566MgWb1cLnmzJYsr161+Y3V+5l+a4jBAVY+evPhzRr0DkZquygyo6IiDQfwzAoKq8iPKj2nZub286sQjbsz+PyoZ3qPG3+T/O28M/laXSJCeHre87lb8t288KCnQA8OnkAN5/mcR+noqF/fyvsoLAjIiJSn6LyKsb/dSmZBWV0igo2Dyy9+axuPHLpAKwnuWFjY9A0loiIiDSaMIedRy9zn/V1MK+UQJuVZ382mEcvG9giQedkaDWWiIiINMiEgQlcf2YX1u3L48krBx23KWJrpbAjIiIiDWKxWPjTFSktPYyTpmksERER8WsKOyIiIuLXFHZERETErynsiIiIiF9T2BERERG/prAjIiIifk1hR0RERPyawo6IiIj4NYUdERER8WsKOyIiIuLXFHZERETErynsiIiIiF9T2BERERG/prAjIiIifs3e0gNoDQzDAKCgoKCFRyIiIiIN5f172/v3+Iko7ACFhYUAJCcnt/BIRERE5GQVFhYSGRl5wtctRn1xqB1wuVwcOnSI8PBwLBZLSw+nURUUFJCcnMz+/fuJiIho6eE0ufZ0v+3pXkH36+/a0/22p3uFpr1fwzAoLCwkKSkJq/XEnTmq7ABWq5XOnTu39DCaVERERLv4P5VXe7rf9nSvoPv1d+3pftvTvULT3W9dFR0vNSiLiIiIX1PYEREREb+msOPnHA4Hf/zjH3E4HC09lGbRnu63Pd0r6H79XXu63/Z0r9A67lcNyiIiIuLXVNkRERERv6awIyIiIn5NYUdERET8msKOiIiI+DWFHT8wc+ZMRo0aRXh4OHFxcVxxxRVs377d55qysjKmTZtGbGwsYWFhTJkyhaysrBYaceN6+umnsVgs3HPPPeZz/na/Bw8e5Prrryc2Npbg4GBSUlL44YcfzNcNw+CRRx4hMTGR4OBgxo8fz86dO1twxKfG6XTy8MMP0717d4KDg+nZsydPPPGEz7k3bflely1bxuTJk0lKSsJisfDxxx/7vN6Qe8vNzWXq1KlEREQQFRXFLbfcQlFRUTPeRcPVdb+VlZU8+OCDpKSkEBoaSlJSEjfeeCOHDh3yeQ9/ud9j3XHHHVgsFl544QWf59vK/TbkXrdu3cpll11GZGQkoaGhjBo1ivT0dPP15vw9rbDjB5YuXcq0adNYtWoV8+fPp7Kykosuuoji4mLzmhkzZvDZZ5/xwQcfsHTpUg4dOsRVV13VgqNuHGvWrOFvf/sbgwcP9nnen+736NGjjBkzhoCAAL788ku2bNnCX//6V6Kjo81rnn32WV566SVee+01Vq9eTWhoKBMmTKCsrKwFR37ynnnmGWbPns0rr7zC1q1beeaZZ3j22Wd5+eWXzWva8r0WFxczZMgQZs2aVevrDbm3qVOnsnnzZubPn8+8efNYtmwZt99+e3Pdwkmp635LSkpYt24dDz/8MOvWrWPOnDls376dyy67zOc6f7nfmubOncuqVatISko67rW2cr/13evu3bs5++yz6devH0uWLGHjxo08/PDDBAUFmdc06+9pQ/xOdna2ARhLly41DMMw8vLyjICAAOODDz4wr9m6dasBGCtXrmypYZ62wsJCo3fv3sb8+fONsWPHGnfffbdhGP53vw8++KBx9tlnn/B1l8tlJCQkGH/+85/N5/Ly8gyHw2H873//a44hNppLLrnE+NWvfuXz3FVXXWVMnTrVMAz/ulfAmDt3rvm4Ife2ZcsWAzDWrFljXvPll18aFovFOHjwYLON/VQce7+1+f777w3A2Ldvn2EY/nm/Bw4cMDp16mT89NNPRteuXY3nn3/efK2t3m9t9/qLX/zCuP7660/4M839e1qVHT+Un58PQExMDABr166lsrKS8ePHm9f069ePLl26sHLlyhYZY2OYNm0al1xyic99gf/d76effsrIkSP5+c9/TlxcHMOGDeMf//iH+XpaWhqZmZk+9xsZGcno0aPb3P2eddZZLFy4kB07dgDw448/snz5ci6++GLAv+71WA25t5UrVxIVFcXIkSPNa8aPH4/VamX16tXNPubGlp+fj8ViISoqCvC/+3W5XNxwww088MADDBw48LjX/eV+XS4Xn3/+OX369GHChAnExcUxevRon6mu5v49rbDjZ1wuF/fccw9jxoxh0KBBAGRmZhIYGGj+AvGKj48nMzOzBUZ5+t59913WrVvHzJkzj3vN3+53z549zJ49m969e/P1119z5513ctddd/Hmm28CmPcUHx/v83Nt8X5/97vfcc0119CvXz8CAgIYNmwY99xzD1OnTgX8616P1ZB7y8zMJC4uzud1u91OTExMm7//srIyHnzwQa699lrzsEh/u99nnnkGu93OXXfdVevr/nK/2dnZFBUV8fTTTzNx4kS++eYbrrzySq666iqWLl0KNP/vaZ167memTZvGTz/9xPLly1t6KE1m//793H333cyfP99n/tdfuVwuRo4cyVNPPQXAsGHD+Omnn3jttde46aabWnh0jev999/n7bff5p133mHgwIFs2LCBe+65h6SkJL+7V6lWWVnJ1VdfjWEYzJ49u6WH0yTWrl3Liy++yLp167BYLC09nCblcrkAuPzyy5kxYwYAQ4cOZcWKFbz22muMHTu22cekyo4fmT59OvPmzWPx4sV07tzZfD4hIYGKigry8vJ8rs/KyiIhIaGZR3n61q5dS3Z2NsOHD8dut2O321m6dCkvvfQSdrud+Ph4v7rfxMREBgwY4PNc//79zVUN3ns6dhVDW7zfBx54wKzupKSkcMMNNzBjxgyzgudP93qshtxbQkIC2dnZPq9XVVWRm5vbZu/fG3T27dvH/PnzzaoO+Nf9fvvtt2RnZ9OlSxfz99a+ffu477776NatG+A/99uhQwfsdnu9v7ea8/e0wo4fMAyD6dOnM3fuXBYtWkT37t19Xh8xYgQBAQEsXLjQfG779u2kp6eTmpra3MM9bRdccAGbNm1iw4YN5tfIkSOZOnWq+b0/3e+YMWOO20pgx44ddO3aFYDu3buTkJDgc78FBQWsXr26zd1vSUkJVqvvryWbzWb+l6I/3euxGnJvqamp5OXlsXbtWvOaRYsW4XK5GD16dLOP+XR5g87OnTtZsGABsbGxPq/70/3ecMMNbNy40ef3VlJSEg888ABff/014D/3GxgYyKhRo+r8vdXsfy81esuzNLs777zTiIyMNJYsWWJkZGSYXyUlJeY1d9xxh9GlSxdj0aJFxg8//GCkpqYaqampLTjqxlVzNZZh+Nf9fv/994bdbjeefPJJY+fOncbbb79thISEGP/973/Na55++mkjKirK+OSTT4yNGzcal19+udG9e3ejtLS0BUd+8m666SajU6dOxrx584y0tDRjzpw5RocOHYzf/va35jVt+V4LCwuN9evXG+vXrzcA47nnnjPWr19vrj5qyL1NnDjRGDZsmLF69Wpj+fLlRu/evY1rr722pW6pTnXdb0VFhXHZZZcZnTt3NjZs2ODzu6u8vNx8D3+539ocuxrLMNrO/dZ3r3PmzDECAgKMv//978bOnTuNl19+2bDZbMa3335rvkdz/p5W2PEDQK1fr7/+unlNaWmp8etf/9qIjo42QkJCjCuvvNLIyMhouUE3smPDjr/d72effWYMGjTIcDgcRr9+/Yy///3vPq+7XC7j4YcfNuLj4w2Hw2FccMEFxvbt21totKeuoKDAuPvuu40uXboYQUFBRo8ePYzf//73Pn/5teV7Xbx4ca3/X73pppsMw2jYveXk5BjXXnutERYWZkRERBi//OUvjcLCwha4m/rVdb9paWkn/N21ePFi8z385X5rU1vYaSv325B7/de//mX06tXLCAoKMoYMGWJ8/PHHPu/RnL+nLYZRY2tSERERET+jnh0RERHxawo7IiIi4tcUdkRERMSvKeyIiIiIX1PYEREREb+msCMiIiJ+TWFHRERE/JrCjoiIiPg1hR0RaRHdunXjhRdeaPD1S5YswWKxHHdwoIhIfbSDsog0yLhx4xg6dOhJBZS6HD58mNDQUEJCQhp0fUVFBbm5ucTHx2OxWBplDCdryZIlnHfeeRw9epSoqKgWGYOInDx7Sw9ARPyHYRg4nU7s9vp/tXTs2PGk3jswMJCEhIRTHZqItGOaxhKRet18880sXbqUF198EYvFgsViYe/evebU0pdffsmIESNwOBwsX76c3bt3c/nllxMfH09YWBijRo1iwYIFPu957DSWxWLhn//8J1deeSUhISH07t2bTz/91Hz92GmsN954g6ioKL7++mv69+9PWFgYEydOJCMjw/yZqqoq7rrrLqKiooiNjeXBBx/kpptu4oorrjjhve7bt4/JkycTHR1NaGgoAwcO5IsvvmDv3r2cd955AERHR2OxWLj55psBcLlczJw5k+7duxMcHMyQIUP48MMPjxv7559/zuDBgwkKCuLMM8/kp59+qvdzReT0KeyISL1efPFFUlNTue2228jIyCAjI4Pk5GTz9d/97nc8/fTTbN26lcGDB1NUVMSkSZNYuHAh69evZ+LEiUyePJn09PQ6P+exxx7j6quvZuPGjUyaNImpU6eSm5t7wutLSkr4y1/+wn/+8x+WLVtGeno6999/v/n6M888w9tvv83rr7/Od999R0FBAR9//HGdY5g2bRrl5eUsW7aMTZs28cwzzxAWFkZycjIfffQRANu3bycjI4MXX3wRgJkzZ/LWW2/x2muvsXnzZmbMmMH111/P0qVLfd77gQce4K9//Str1qyhY8eOTJ48mcrKyjo/V0QaQZOcpS4ifmfs2LHG3Xff7fPc4sWLDcD4+OOP6/35gQMHGi+//LL5uGvXrsbzzz9vPgaMP/zhD+bjoqIiAzC+/PJLn886evSoYRiG8frrrxuAsWvXLvNnZs2aZcTHx5uP4+PjjT//+c/m46qqKqNLly7G5ZdffsJxpqSkGI8++mitrx07BsMwjLKyMiMkJMRYsWKFz7W33HKLce211/r83Lvvvmu+npOTYwQHBxvvvfdevZ8rIqdHPTsictpGjhzp87ioqIhHH32Uzz//nIyMDKqqqigtLa23sjN48GDz+9DQUCIiIsjOzj7h9SEhIfTs2dN8nJiYaF6fn59PVlYWZ5xxhvm6zWZjxIgRuFyuE77nXXfdxZ133sk333zD+PHjmTJlis+4jrVr1y5KSkq48MILfZ6vqKhg2LBhPs+lpqaa38fExNC3b1+2bt16Sp8rIg2naSwROW2hoaE+j++//37mzp3LU089xbfffsuGDRtISUmhoqKizvcJCAjweWyxWOoMJrVdb5zmAtNbb72VPXv2cMMNN7Bp0yZGjhzJyy+/fMLri4qKAPj888/ZsGGD+bVlyxafvp3G/lwRaTiFHRFpkMDAQJxOZ4Ou/e6777j55pu58sorSUlJISEhgb179zbtAI8RGRlJfHw8a9asMZ9zOp2sW7eu3p9NTk7mjjvuYM6cOdx333384x//ANz/DLzv4zVgwAAcDgfp6en06tXL56tmXxPAqlWrzO+PHj3Kjh076N+/f72fKyKnR9NYItIg3bp1Y/Xq1ezdu5ewsDBiYmJOeG3v3r2ZM2cOkydPxmKx8PDDD9dZoWkqv/nNb5g5cya9evWiX79+vPzyyxw9erTOfXruueceLr74Yvr06cPRo0dZvHixGUi6du2KxWJh3rx5TJo0ieDgYMLDw7n//vuZMWMGLpeLs88+m/z8fL777jsiIiK46aabzPd+/PHHiY2NJT4+nt///vd06NDBXBlW1+eKyOlRZUdEGuT+++/HZrMxYMAAOnbsWGf/zXPPPUd0dDRnnXUWkydPZsKECQwfPrwZR+v24IMPcu2113LjjTeSmppKWFgYEyZMICgo6IQ/43Q6mTZtGv3792fixIn06dOHV199FYBOnTrx2GOP8bvf/Y74+HimT58OwBNPPMHDDz/MzJkzzZ/7/PPP6d69u897P/3009x9992MGDGCzMxMPvvsM59q0Yk+V0ROj3ZQFpF2w+Vy0b9/f66++mqeeOKJZvtc7bws0rI0jSUifmvfvn188803jB07lvLycl555RXS0tK47rrrWnpoItKMNI0lIn7LarXyxhtvMGrUKMaMGcOmTZtYsGCBemFE2hlNY4mIiIhfU2VHRERE/JrCjoiIiPg1hR0RERHxawo7IiIi4tcUdkRERMSvKeyIiIiIX1PYEREREb+msCMiIiJ+7f8DWMzTqMqtQGYAAAAASUVORK5CYII=",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# 准备优化器和模型\n",
    "vocab_size = len(dataset.token2id)\n",
    "hidden_size = 32\n",
    "n_layers = 1\n",
    "dropout = 0\n",
    "n_tags = len(dataset.label2id)\n",
    "batch_size = 128\n",
    "epochs = 20\n",
    "learning_rate = 1e-2\n",
    "\n",
    "lstm_crf = LSTM_CRF(vocab_size, hidden_size, n_layers, dropout, n_tags)\n",
    "train(lstm_crf, batch_size, epochs, learning_rate)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "8a9138a5",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "precision = 0.5678384442782348, recall = 0.452551263710062, f1 = 0.5036820805413653\n",
      "precision = 0.5174089068825911, recall = 0.391304347826087, f1 = 0.4456066945606695\n"
     ]
    }
   ],
   "source": [
    "evaluate(train_X, train_Y, lstm_crf, batch_size)\n",
    "evaluate(test_X, test_Y, lstm_crf, batch_size)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c0dcde75-97c9-480e-9da2-efd58d7f7a46",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "sun",
   "language": "python",
   "name": "sun"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.12.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
